diff --git a/.buildkite/debug.yml b/.buildkite/debug.yml index 9c322aa96f7c6f666ad787f0ee7f1ee6c361dcac..209cf2759a6e25deeeec3c30c4aec13d629ba56c 100644 --- a/.buildkite/debug.yml +++ b/.buildkite/debug.yml @@ -96,7 +96,7 @@ steps: echo "--- :m: Starting MongoDB" && \ $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \ echo "+++ :microscope: Running tests" && \ - ln -s "$(pwd)" /data/job && cd /data/job/build && ctest --output-on-failure + ln -s "$(pwd)" /data/job && cd /data/job/build && ctest -LE long_runing_tests --output-on-failure retry: automatic: limit: 1 @@ -116,7 +116,7 @@ steps: echo "--- :m: Starting MongoDB" && \ $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \ echo "+++ :microscope: Running tests" && \ - cd /data/job/build && ctest --output-on-failure + cd /data/job/build && ctest -LE long_runing_tests --output-on-failure retry: automatic: limit: 1 @@ -140,7 +140,7 @@ steps: echo "--- :m: Starting MongoDB" && \ $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \ echo "+++ :microscope: Running tests" && \ - cd /data/job/build && ctest --output-on-failure + cd /data/job/build && ctest -LE long_runing_tests --output-on-failure retry: automatic: limit: 1 @@ -164,7 +164,7 @@ steps: echo "--- :m: Starting MongoDB" && \ $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \ echo "+++ :microscope: Running tests" && \ - cd /data/job/build && ctest --output-on-failure + cd /data/job/build && ctest -LE long_runing_tests --output-on-failure retry: automatic: limit: 1 @@ -188,7 +188,7 @@ steps: echo "--- :m: Starting MongoDB" && \ $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \ echo "+++ :microscope: Running tests" && \ - cd /data/job/build && ctest --output-on-failure + cd /data/job/build && ctest -LE long_runing_tests --output-on-failure retry: automatic: limit: 1 @@ -212,7 +212,7 @@ steps: echo "--- :m: Starting MongoDB" && \ $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \ echo "+++ :microscope: Running tests" && \ - cd /data/job/build && ctest --output-on-failure + cd /data/job/build && ctest -LE long_runing_tests --output-on-failure retry: automatic: limit: 1 diff --git a/.buildkite/long_running_tests.yml b/.buildkite/long_running_tests.yml new file mode 100644 index 0000000000000000000000000000000000000000..d992980866a08946315ab5eb7defdd3413d4a3c0 --- /dev/null +++ b/.buildkite/long_running_tests.yml @@ -0,0 +1,230 @@ +steps: + - command: | + echo "--- Creating symbolic link to job directory :file_folder:" && \ + sleep 5 && ln -s "$(pwd)" /data/job && cd /data/job && \ + echo "+++ Building :hammer:" && \ + echo 1 | ./eosio_build.sh && \ + echo "--- Compressing build directory :compression:" && \ + tar -pczf build.tar.gz build/ + label: ":darwin: Build" + agents: + - "role=macos-builder" + artifact_paths: "build.tar.gz" + timeout: 60 + + - command: | + echo "+++ :hammer: Building" && \ + echo 1 | ./eosio_build.sh && \ + echo "--- :compression: Compressing build directory" && \ + tar -pczf build.tar.gz build/ + label: ":ubuntu: Build" + agents: + - "role=linux-builder" + artifact_paths: "build.tar.gz" + plugins: + docker#v1.1.1: + image: "eosio/ci:ubuntu" + workdir: /data/job + timeout: 60 + + - command: | + echo "+++ :hammer: Building" && \ + echo 1 | ./eosio_build.sh && \ + echo "--- :compression: Compressing build directory" && \ + tar -pczf build.tar.gz build/ + label: ":ubuntu: 18.04 Build" + agents: + - "role=linux-builder" + artifact_paths: "build.tar.gz" + plugins: + docker#v1.1.1: + image: "eosio/ci:ubuntu18" + workdir: /data/job + timeout: 60 + + - command: | + echo "+++ :hammer: Building" && \ + echo 1 | ./eosio_build.sh && \ + echo "--- :compression: Compressing build directory" && \ + tar -pczf build.tar.gz build/ + label: ":fedora: Build" + agents: + - "role=linux-builder" + artifact_paths: "build.tar.gz" + plugins: + docker#v1.1.1: + image: "eosio/ci:fedora" + workdir: /data/job + timeout: 60 + + - command: | + echo "+++ :hammer: Building" && \ + echo 1 | ./eosio_build.sh && \ + echo "--- :compression: Compressing build directory" && \ + tar -pczf build.tar.gz build/ + label: ":centos: Build" + agents: + - "role=linux-builder" + artifact_paths: "build.tar.gz" + plugins: + docker#v1.1.1: + image: "eosio/ci:centos" + workdir: /data/job + timeout: 60 + + - command: | + echo "+++ :hammer: Building" && \ + echo 1 | ./eosio_build.sh && \ + echo "--- :compression: Compressing build directory" && \ + tar -pczf build.tar.gz build/ + label: ":aws: Build" + agents: + - "role=linux-builder" + artifact_paths: "build.tar.gz" + plugins: + docker#v1.1.1: + image: "eosio/ci:amazonlinux" + workdir: /data/job + timeout: 60 + + - wait + + - command: | + echo "--- :arrow_down: Downloading build directory" && \ + buildkite-agent artifact download "build.tar.gz" . --step ":darwin: Build" && + tar -zxf build.tar.gz && \ + echo "--- :m: Starting MongoDB" && \ + $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \ + echo "+++ :microscope: Running tests" && \ + ln -s "$(pwd)" /data/job && cd /data/job/build && ctest -L long_runing_tests --output-on-failure + retry: + automatic: + limit: 1 + label: ":darwin: Tests" + agents: + - "role=macos-tester" + artifact_paths: + - "mongod.log" + - "build/genesis.json" + - "build/config.ini" + timeout: 60 + + - command: | + echo "--- :arrow_down: Downloading build directory" && \ + buildkite-agent artifact download "build.tar.gz" . --step ":ubuntu: Build" && + tar -zxf build.tar.gz && \ + echo "--- :m: Starting MongoDB" && \ + $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \ + echo "+++ :microscope: Running tests" && \ + cd /data/job/build && ctest -L long_runing_tests --output-on-failure + retry: + automatic: + limit: 1 + label: ":ubuntu: Tests" + agents: + - "role=linux-tester" + artifact_paths: + - "mongod.log" + - "build/genesis.json" + - "build/config.ini" + plugins: + docker#v1.1.1: + image: "eosio/ci:ubuntu" + workdir: /data/job + timeout: 60 + + - command: | + echo "--- :arrow_down: Downloading build directory" && \ + buildkite-agent artifact download "build.tar.gz" . --step ":ubuntu: 18.04 Build" && + tar -zxf build.tar.gz && \ + echo "--- :m: Starting MongoDB" && \ + $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \ + echo "+++ :microscope: Running tests" && \ + cd /data/job/build && ctest -L long_runing_tests --output-on-failure + retry: + automatic: + limit: 1 + label: ":ubuntu: 18.04 Tests" + agents: + - "role=linux-tester" + artifact_paths: + - "mongod.log" + - "build/genesis.json" + - "build/config.ini" + plugins: + docker#v1.1.1: + image: "eosio/ci:ubuntu18" + workdir: /data/job + timeout: 60 + + - command: | + echo "--- :arrow_down: Downloading build directory" && \ + buildkite-agent artifact download "build.tar.gz" . --step ":fedora: Build" && + tar -zxf build.tar.gz && \ + echo "--- :m: Starting MongoDB" && \ + $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \ + echo "+++ :microscope: Running tests" && \ + cd /data/job/build && ctest -L long_runing_tests --output-on-failure + retry: + automatic: + limit: 1 + label: ":fedora: Tests" + agents: + - "role=linux-tester" + artifact_paths: + - "mongod.log" + - "build/genesis.json" + - "build/config.ini" + plugins: + docker#v1.1.1: + image: "eosio/ci:fedora" + workdir: /data/job + timeout: 60 + + - command: | + echo "--- :arrow_down: Downloading build directory" && \ + buildkite-agent artifact download "build.tar.gz" . --step ":centos: Build" && + tar -zxf build.tar.gz && \ + echo "--- :m: Starting MongoDB" && \ + $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \ + echo "+++ :microscope: Running tests" && \ + cd /data/job/build && ctest -L long_runing_tests --output-on-failure + retry: + automatic: + limit: 1 + label: ":centos: Tests" + agents: + - "role=linux-tester" + artifact_paths: + - "mongod.log" + - "build/genesis.json" + - "build/config.ini" + plugins: + docker#v1.1.1: + image: "eosio/ci:centos" + workdir: /data/job + timeout: 60 + + - command: | + echo "--- :arrow_down: Downloading build directory" && \ + buildkite-agent artifact download "build.tar.gz" . --step ":aws: Build" && + tar -zxf build.tar.gz && \ + echo "--- :m: Starting MongoDB" && \ + $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \ + echo "+++ :microscope: Running tests" && \ + cd /data/job/build && ctest -L long_runing_tests --output-on-failure + retry: + automatic: + limit: 1 + label: ":aws: Tests" + agents: + - "role=linux-tester" + artifact_paths: + - "mongod.log" + - "build/genesis.json" + - "build/config.ini" + plugins: + docker#v1.1.1: + image: "eosio/ci:amazonlinux" + workdir: /data/job + timeout: 60 diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index ca662111cd000d443c1cb7366c15448c742cd028..e542210ca8961b6dddff709247ec96789b6b2a70 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -96,7 +96,7 @@ steps: echo "--- :m: Starting MongoDB" && \ $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \ echo "+++ :microscope: Running tests" && \ - ln -s "$(pwd)" /data/job && cd /data/job/build && ctest --output-on-failure + ln -s "$(pwd)" /data/job && cd /data/job/build && ctest -LE long_runing_tests --output-on-failure retry: automatic: limit: 1 @@ -116,7 +116,7 @@ steps: echo "--- :m: Starting MongoDB" && \ $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \ echo "+++ :microscope: Running tests" && \ - cd /data/job/build && ctest --output-on-failure + cd /data/job/build && ctest -LE long_runing_tests --output-on-failure retry: automatic: limit: 1 @@ -140,7 +140,7 @@ steps: echo "--- :m: Starting MongoDB" && \ $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \ echo "+++ :microscope: Running tests" && \ - cd /data/job/build && ctest --output-on-failure + cd /data/job/build && ctest -LE long_runing_tests --output-on-failure retry: automatic: limit: 1 @@ -164,7 +164,7 @@ steps: echo "--- :m: Starting MongoDB" && \ $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \ echo "+++ :microscope: Running tests" && \ - cd /data/job/build && ctest --output-on-failure + cd /data/job/build && ctest -LE long_runing_tests --output-on-failure retry: automatic: limit: 1 @@ -188,7 +188,7 @@ steps: echo "--- :m: Starting MongoDB" && \ $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \ echo "+++ :microscope: Running tests" && \ - cd /data/job/build && ctest --output-on-failure + cd /data/job/build && ctest -LE long_runing_tests --output-on-failure retry: automatic: limit: 1 @@ -212,7 +212,7 @@ steps: echo "--- :m: Starting MongoDB" && \ $(which mongod) --fork --logpath "$(pwd)"/mongod.log && \ echo "+++ :microscope: Running tests" && \ - cd /data/job/build && ctest --output-on-failure + cd /data/job/build && ctest -LE long_runing_tests --output-on-failure retry: automatic: limit: 1 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e580228aecd5511ec6a20c6e7d71bc080c241422..2c8bfb4556804825a5f819cf6b06da744cc96292 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -50,7 +50,6 @@ add_test(NAME p2p_dawn515_test COMMAND tests/p2p_tests/dawn_515/test.sh WORKING_ # add_test(NAME nodeos_run_test-mongodb COMMAND tests/nodeos_run_test.py --mongodb -v --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) #endif() -# TODO: Tests removed until working again on master. add_test(NAME distributed-transactions-test COMMAND tests/distributed-transactions-test.py -d 2 -p 1 -n 4 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) add_test(NAME restart-scenarios-test-resync COMMAND tests/restart-scenarios-test.py -c resync -p4 -v --clean-run --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) add_test(NAME restart-scenarios-test-hard_replay COMMAND tests/restart-scenarios-test.py -c hardReplay -p4 -v --clean-run --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) @@ -58,6 +57,12 @@ add_test(NAME restart-scenarios-test-none COMMAND tests/restart-scenarios-test.p # TODO: add_test(NAME consensus-validation-malicious-producers COMMAND tests/consensus-validation-malicious-producers.py -w 80 --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) add_test(NAME validate_dirty_db_test COMMAND tests/validate-dirty-db.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +# Long running tests +add_test(NAME nodeos_sanity_lr_test COMMAND tests/nodeos_run_test.py -v --sanity-test --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +#add_test(NAME distributed_transactions_lr_test COMMAND tests/distributed-transactions-test.py -d 2 -p 21 -n 21 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +set_property(TEST nodeos_sanity_lr_test PROPERTY LABELS long_runing_tests) +#set_property(TEST distributed_transactions_lr_test PROPERTY LABELS long_runing_tests) + if(ENABLE_COVERAGE_TESTING) set(Coverage_NAME ${PROJECT_NAME}_coverage) diff --git a/tests/Cluster.py b/tests/Cluster.py index a0c98734034d3edebc233e0bfb069c23c32f2233..fe2a203eb40e9b18ce6c491aafca95bc99e7b0ff 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -13,6 +13,7 @@ import sys import random import json import socket +import errno from core_symbol import CORE_SYMBOL from testUtils import Utils @@ -32,7 +33,7 @@ class Cluster(object): # pylint: disable=too-many-arguments # walletd [True|False] Is keosd running. If not load the wallet plugin - def __init__(self, walletd=False, localCluster=True, host="localhost", port=8888, walletHost="localhost", walletPort=8899, enableMongo=False + def __init__(self, walletd=False, localCluster=True, host="localhost", port=8888, walletHost="localhost", walletPort=9899, enableMongo=False , mongoHost="localhost", mongoPort=27017, mongoDb="EOStest", defproduceraPrvtKey=None, defproducerbPrvtKey=None, staging=False): """Cluster container. walletd [True|False] Is wallet keosd running. If not load the wallet plugin @@ -191,7 +192,7 @@ class Cluster(object): account.activePrivateKey=keys["private"] account.activePublicKey=keys["public"] - for name,initKeys in producerKeys.items(): + for name,_ in producerKeys.items(): account=Account(name) initAccountKeys(account, producerKeys[name]) self.defProducerAccounts[name] = account @@ -322,7 +323,7 @@ class Cluster(object): @staticmethod def getClientVersion(verbose=False): """Returns client version (string)""" - p = re.compile('^Build version:\s(\w+)\n$') + p = re.compile(r'^Build version:\s(\w+)\n$') try: cmd="%s version client" % (Utils.EosClientPath) if verbose: Utils.Print("cmd: %s" % (cmd)) @@ -476,7 +477,7 @@ class Cluster(object): node=self.nodes[nextEosIdx] if Utils.Debug: Utils.Print("Wait for transaction id %s on node port %d" % (transId, node.port)) if node.waitForTransInBlock(transId) is False: - Utils.Print("ERROR: Selected node never received transaction id %s" % (transId)) + Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, node.port)) return False transferAmount -= amount @@ -497,7 +498,7 @@ class Cluster(object): node=self.nodes[0] if Utils.Debug: Utils.Print("Wait for transaction id %s on node port %d" % (transId, node.port)) if node.waitForTransInBlock(transId) is False: - Utils.Print("ERROR: Selected node never received transaction id %s" % (transId)) + Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, node.port)) return False return True @@ -728,7 +729,9 @@ class Cluster(object): accounts.append(initx) transId=Node.getTransId(trans) - biosNode.waitForTransInBlock(transId) + if not biosNode.waitForTransInBlock(transId): + Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port)) + return False Utils.Print("Validating system accounts within bootstrap") biosNode.validateAccounts(accounts) @@ -776,6 +779,7 @@ class Cluster(object): trans=trans[1] transId=Node.getTransId(trans) if not biosNode.waitForTransInBlock(transId): + Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port)) return False # wait for block production handover (essentially a block produced by anyone but eosio). @@ -815,7 +819,9 @@ class Cluster(object): Node.validateTransaction(trans) transId=Node.getTransId(trans) - biosNode.waitForTransInBlock(transId) + if not biosNode.waitForTransInBlock(transId): + Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port)) + return False contract="eosio.token" contractDir="contracts/%s" % (contract) @@ -840,7 +846,9 @@ class Cluster(object): Node.validateTransaction(trans[1]) transId=Node.getTransId(trans[1]) - biosNode.waitForTransInBlock(transId) + if not biosNode.waitForTransInBlock(transId): + Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port)) + return False contract=eosioTokenAccount.name Utils.Print("push issue action to %s contract" % (contract)) @@ -856,7 +864,11 @@ class Cluster(object): Utils.Print("Wait for issue action transaction to become finalized.") transId=Node.getTransId(trans[1]) # biosNode.waitForTransInBlock(transId) - biosNode.waitForTransFinalization(transId) + # guesstimating block finalization timeout. Two production rounds of 12 blocks per node, plus 60 seconds buffer + timeout = .5 * 12 * 2 * len(producerKeys) + 60 + if not biosNode.waitForTransFinalization(transId, timeout=timeout): + Utils.Print("ERROR: Failed to validate transaction %s got rolled into a finalized block on server port %d." % (transId, biosNode.port)) + return False expectedAmount="1000000000.0000 {0}".format(CORE_SYMBOL) Utils.Print("Verify eosio issue, Expected: %s" % (expectedAmount)) @@ -896,6 +908,7 @@ class Cluster(object): Utils.Print("Wait for last transfer transaction to become finalized.") transId=Node.getTransId(trans[1]) if not biosNode.waitForTransInBlock(transId): + Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port)) return False Utils.Print("Cluster bootstrap done.") @@ -1073,8 +1086,7 @@ class Cluster(object): node=self.nodes[0] if Utils.Debug: Utils.Print("Wait for transaction id %s on server port %d." % ( transId, node.port)) if node.waitForTransInBlock(transId) is False: - Utils.Print("ERROR: Failed waiting for transaction id %s on server port %d." % ( - transId, node.port)) + Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, node.port)) return False return True diff --git a/tests/Node.py b/tests/Node.py index 6709670a6019394b04bde1648fec56d6e182f00c..c938c97cde6035db9cc8f9f9ec17cf9190bb9d93 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -228,7 +228,14 @@ class Node(object): Utils.Print("Failure in get info parsing. %s" % (info)) raise - return True if blockNum <= node_block_num else False + finalized = True if blockNum <= node_block_num else False + if Utils.Debug: + if finalized: + Utils.Print("Block %d is finalized." % (blockNum)) + else: + Utils.Print("Block %d is not yet finalized." % (blockNum)) + + return finalized # pylint: disable=too-many-branches def getTransaction(self, transId, retry=True, silentErrors=False): @@ -314,10 +321,13 @@ class Node(object): try: headBlockNum=int(headBlockNum) except(ValueError) as _: - Utils.Print("Info parsing failed. %s" % (headBlockNum)) + Utils.Print("ERROR: Block info parsing failed. %s" % (headBlockNum)) + raise + if Utils.Debug: Utils.Print("Reference block num %d, Head block num: %d" % (refBlockNum, headBlockNum)) for blockNum in range(refBlockNum, headBlockNum+1): if self.isTransInBlock(str(transId), blockNum): + if Utils.Debug: Utils.Print("Found transaction %s in block %d" % (transId, blockNum)) return blockNum return None diff --git a/tests/WalletMgr.py b/tests/WalletMgr.py index 6a7009b59f51062d1b54fc71e011cd751a5291e2..8cb8020dad4325cd5ede242b65ebd1b4a25b23c9 100644 --- a/tests/WalletMgr.py +++ b/tests/WalletMgr.py @@ -17,7 +17,7 @@ class WalletMgr(object): # pylint: disable=too-many-arguments # walletd [True|False] True=Launch wallet(keosd) process; False=Manage launch process externally. - def __init__(self, walletd, nodeosPort=8888, nodeosHost="localhost", port=8899, host="localhost"): + def __init__(self, walletd, nodeosPort=8888, nodeosHost="localhost", port=9899, host="localhost"): self.walletd=walletd self.nodeosPort=nodeosPort self.nodeosHost=nodeosHost