diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9739faab7272e72e01af726ad0bde88d7b67b0ea..ced40eadcc5f503673357a9945173c01707bcd1d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -38,8 +38,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/validate-dirty-db.py ${CMAKE_CURRENT_ #To run plugin_test with all log from blockchain displayed, put --verbose after --, i.e. plugin_test -- --verbose add_test(NAME plugin_test COMMAND plugin_test --report_level=detailed --color_output) -add_test(NAME nodeos_run_test COMMAND tests/nodeos_run_test.py -v --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -add_test(NAME nodeos_run_remote_test COMMAND tests/nodeos_run_remote_test.py -v --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME nodeos_run_test COMMAND tests/nodeos_run_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) add_test(NAME p2p_dawn515_test COMMAND tests/p2p_tests/dawn_515/test.sh WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) #if(BUILD_MONGO_DB_PLUGIN) @@ -47,13 +46,11 @@ add_test(NAME p2p_dawn515_test COMMAND tests/p2p_tests/dawn_515/test.sh WORKING_ #endif() # TODO: Tests removed until working again on master. -add_test(NAME distributed-transactions-test COMMAND tests/distributed-transactions-test.py -d 2 -p 1 -n 4 -v --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -add_test(NAME distributed-transactions-remote-test COMMAND tests/distributed-transactions-remote-test.py -v --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -# TODO removed on slim: add_test(NAME restart-scenarios-test_resync COMMAND tests/restart-scenarios-test.py -c resync -p4 -v --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -# add_test(NAME restart-scenarios-test_replay COMMAND tests/restart-scenarios-test.py -c replay -p4 -v --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -add_test(NAME restart-scenarios-test_hard_replay COMMAND tests/restart-scenarios-test.py -c hardReplay -p4 -v --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME distributed-transactions-test COMMAND tests/distributed-transactions-test.py -d 2 -p 1 -n 4 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME restart-scenarios-test-resync COMMAND tests/restart-scenarios-test.py -c resync -p4 -v --clean-run --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME restart-scenarios-test-hard_replay COMMAND tests/restart-scenarios-test.py -c hardReplay -p4 -v --clean-run --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) # TODO: add_test(NAME consensus-validation-malicious-producers COMMAND tests/consensus-validation-malicious-producers.py -w 80 --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) -add_test(NAME validate_dirty_db_test COMMAND tests/validate-dirty-db.py -v --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME validate_dirty_db_test COMMAND tests/validate-dirty-db.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) if(ENABLE_COVERAGE_TESTING) diff --git a/tests/consensus-validation-malicious-producers.py b/tests/consensus-validation-malicious-producers.py index e59af720da77d8424be5cc0a6067726b6f3944f2..b31fd8ba4554ddc1021b9912efd5a706c843ccf5 100755 --- a/tests/consensus-validation-malicious-producers.py +++ b/tests/consensus-validation-malicious-producers.py @@ -333,7 +333,7 @@ def myTest(transWillEnterBlock): transId=testUtils.Node.getTransId(trans[1]) Print("verify transaction exists") - if not node2.waitForTransIdOnNode(transId): + if not node2.waitForTransInBlock(transId): error("Transaction never made it to node2") return False diff --git a/tests/distributed-transactions-remote-test.py b/tests/distributed-transactions-remote-test.py index 27abc040fdd998f71bd9ffad42d5b5e88fc7bc95..5716a47dd049d9e4a1a0ecc6dd586fc8a80c6216 100755 --- a/tests/distributed-transactions-remote-test.py +++ b/tests/distributed-transactions-remote-test.py @@ -7,6 +7,12 @@ import subprocess import tempfile import os +############################################################### +# distributed-transactions-remote-test +# Tests remote capability of the distributed-transactions-test. Test will setup cluster and pass nodes info to distributed-transactions-test. E.g. +# distributed-transactions-remote-test.py -v --clean-run --dump-error-detail +############################################################### + Print=testUtils.Utils.Print def errorExit(msg="", errorCode=1): @@ -18,19 +24,19 @@ pnodes=1 parser = argparse.ArgumentParser() parser.add_argument("-p", type=int, help="producing nodes count", default=pnodes) parser.add_argument("-v", help="verbose", action='store_true') -parser.add_argument("--dont-kill", help="Leave cluster running after test finishes", action='store_true') +parser.add_argument("--leave-running", help="Leave cluster running after test finishes", action='store_true') parser.add_argument("--dump-error-details", help="Upon error print etc/eosio/node_*/config.ini and var/lib/node_*/stderr.log to stdout", action='store_true') -parser.add_argument("--kill-all", help="Kill all nodeos and kleos instances", action='store_true') +parser.add_argument("--clean-run", help="Kill all nodeos and kleos instances", action='store_true') args = parser.parse_args() pnodes=args.p # nodesFile=args.nodes_file debug=args.v -dontKill=args.dont_kill +dontKill=args.leave_running dumpErrorDetails=args.dump_error_details -killAll=args.kill_all +killAll=args.clean_run testUtils.Utils.Debug=debug diff --git a/tests/distributed-transactions-test.py b/tests/distributed-transactions-test.py index 29ebf10b57598cba50c0517f9bd0b3a652768ecf..c4ceba30298a5b0f7abdb93a86c596afd6234e5c 100755 --- a/tests/distributed-transactions-test.py +++ b/tests/distributed-transactions-test.py @@ -21,11 +21,11 @@ parser.add_argument("-s", type=str, help="topology", default="mesh") parser.add_argument("-v", help="verbose", action='store_true') parser.add_argument("--nodes-file", type=str, help="File containing nodes info in JSON format.") parser.add_argument("--seed", type=int, help="random seed", default=seed) -parser.add_argument("--dont-kill", help="Leave cluster running after test finishes", action='store_true') +parser.add_argument("--leave-running", help="Leave cluster running after test finishes", action='store_true') parser.add_argument("--dump-error-details", help="Upon error print etc/eosio/node_*/config.ini and var/lib/node_*/stderr.log to stdout", action='store_true') -parser.add_argument("--kill-all", help="Kill all nodeos and kleos instances", action='store_true') +parser.add_argument("--clean-run", help="Kill all nodeos and kleos instances", action='store_true') args = parser.parse_args() pnodes=args.p @@ -35,9 +35,9 @@ total_nodes = pnodes if args.n == 0 else args.n debug=args.v nodesFile=args.nodes_file seed=args.seed -dontKill=args.dont_kill +dontKill=args.leave_running dumpErrorDetails=args.dump_error_details -killAll=args.kill_all +killAll=args.clean_run killWallet=not dontKill killEosInstances=not dontKill diff --git a/tests/nodeos_run_remote_test.py b/tests/nodeos_run_remote_test.py index 9e0916c12e9a933504f85f16ee96bbbbdf3cc154..c8853e92dad77591c980576a0e6d486d640a3b24 100755 --- a/tests/nodeos_run_remote_test.py +++ b/tests/nodeos_run_remote_test.py @@ -5,6 +5,12 @@ import testUtils import argparse import subprocess +############################################################### +# nodeos_run_remote_test +# Tests remote capability of the nodeos_run_test. Test will setup cluster and pass nodes info to nodeos_run_test. E.g. +# nodeos_run_remote_test.py -v --clean-run --dump-error-detail +############################################################### + Print=testUtils.Utils.Print def errorExit(msg="", errorCode=1): @@ -13,19 +19,19 @@ def errorExit(msg="", errorCode=1): parser = argparse.ArgumentParser() parser.add_argument("-v", help="verbose", action='store_true') -parser.add_argument("--dont-kill", help="Leave cluster running after test finishes", action='store_true') +parser.add_argument("--leave-running", help="Leave cluster running after test finishes", action='store_true') parser.add_argument("--only-bios", help="Limit testing to bios node.", action='store_true') parser.add_argument("--dump-error-details", help="Upon error print etc/eosio/node_*/config.ini and var/lib/node_*/stderr.log to stdout", action='store_true') -parser.add_argument("--kill-all", help="Kill all nodeos and kleos instances", action='store_true') +parser.add_argument("--clean-run", help="Kill all nodeos and kleos instances", action='store_true') args = parser.parse_args() debug=args.v -dontKill=args.dont_kill +dontKill=args.leave_running dumpErrorDetails=args.dump_error_details onlyBios=args.only_bios -killAll=args.kill_all +killAll=args.clean_run testUtils.Utils.Debug=debug diff --git a/tests/nodeos_run_test.py b/tests/nodeos_run_test.py index 535aa53a162dd41bb3a96f18e5803d35a9118726..13fe79dac5792322a525ba27493d768e99681c57 100755 --- a/tests/nodeos_run_test.py +++ b/tests/nodeos_run_test.py @@ -49,9 +49,9 @@ parser.add_argument("--dont-launch", help="Don't launch own node. Assume node is parser.add_argument("--keep-logs", help="Don't delete var/lib/node_* folders upon test completion", action='store_true') parser.add_argument("-v", help="verbose logging", action='store_true') -parser.add_argument("--dont-kill", help="Leave cluster running after test finishes", action='store_true') +parser.add_argument("--leave-running", help="Leave cluster running after test finishes", action='store_true') parser.add_argument("--only-bios", help="Limit testing to bios node.", action='store_true') -parser.add_argument("--kill-all", help="Kill all nodeos and kleos instances", action='store_true') +parser.add_argument("--clean-run", help="Kill all nodeos and kleos instances", action='store_true') args = parser.parse_args() testOutputFile=args.output @@ -64,10 +64,10 @@ defproducerbPrvtKey=args.defproducerb_prvt_key dumpErrorDetails=args.dump_error_details keepLogs=args.keep_logs dontLaunch=args.dont_launch -dontKill=args.dont_kill +dontKill=args.leave_running prodCount=args.prod_count onlyBios=args.only_bios -killAll=args.kill_all +killAll=args.clean_run testUtils.Utils.Debug=debug localTest=True if server == LOCAL_HOST else False @@ -374,7 +374,7 @@ try: # errorExit("FAILURE - %s servants. Expected: [], Actual: %s" % ( # testeraAccount.name, actualServants), raw=True) - node.waitForTransIdOnNode(transId) + node.waitForTransInBlock(transId) transaction=None if not enableMongo: @@ -518,7 +518,7 @@ try: errorExit("Failed to reject duplicate message for currency1111 contract") Print("verify transaction exists") - if not node.waitForTransIdOnNode(transId): + if not node.waitForTransInBlock(transId): cmdError("%s get transaction trans_id" % (ClientName)) errorExit("Failed to verify push message transaction id.") @@ -748,7 +748,7 @@ try: Print("CurrentBlockNum: %d" % (currentBlockNum)) Print("Request blocks 1-%d" % (currentBlockNum)) for blockNum in range(1, currentBlockNum+1): - block=node.getBlock(str(blockNum), retry=False, silentErrors=False) + block=node.getBlock(blockNum, retry=False, silentErrors=False) if block is None: cmdError("%s get block" % (ClientName)) errorExit("get block by num %d" % blockNum) @@ -769,7 +769,7 @@ try: Print("Request invalid block numbered %d. This will generate an expected error message." % (currentBlockNum+1000)) - block=node.getBlock(str(currentBlockNum+1000), silentErrors=True, retry=False) + block=node.getBlock(currentBlockNum+1000, silentErrors=True, retry=False) if block is not None: errorExit("ERROR: Received block where not expected") else: diff --git a/tests/p2p_stress.py b/tests/p2p_stress.py index 3430b3d9ebc84d4d2722c67fd0df2ef4c4ee1224..ef6080d7a07a608be7943f4a5d1e21de3a0e5a71 100644 --- a/tests/p2p_stress.py +++ b/tests/p2p_stress.py @@ -59,7 +59,7 @@ class StressNetwork: if trid is None: return ([], "", 0.0, "failed to issue currency0000") print("transaction id %s" % (trid)) - node.waitForTransIdOnNode(trid) + node.waitForTransInBlock(trid) self.trList = [] expBal = 0 @@ -97,7 +97,7 @@ class StressNetwork: for tr in self.trList: trid = node.getTransId(tr) transIdlist.append(trid) - node.waitForTransIdOnNode(trid) + node.waitForTransInBlock(trid) return (transIdlist, acc2.name, expBal, "") def on_exit(self): diff --git a/tests/restart-scenarios-test.py b/tests/restart-scenarios-test.py index 5c6cb2e2bb8951cfd048dcc59e71528f4150978e..744283f8fa749b6ac3e95c8bb251d2a7eb932def 100755 --- a/tests/restart-scenarios-test.py +++ b/tests/restart-scenarios-test.py @@ -4,6 +4,7 @@ import testUtils import argparse import random +import traceback ############################################################### # Test for different nodes restart scenarios. @@ -25,6 +26,7 @@ Print=testUtils.Utils.Print def errorExit(msg="", errorCode=1): Print("ERROR:", msg) + traceback.print_stack(limit=-1) exit(errorCode) parser = argparse.ArgumentParser() @@ -32,19 +34,20 @@ parser.add_argument("-p", type=int, help="producing nodes count", default=2) parser.add_argument("-d", type=int, help="delay between nodes startup", default=1) parser.add_argument("-s", type=str, help="topology", default="mesh") parser.add_argument("-c", type=str, help="chain strategy[%s|%s|%s]" % - (testUtils.Utils.SyncResyncTag, testUtils.Utils.SyncReplayTag, testUtils.Utils.SyncNoneTag), + (testUtils.Utils.SyncResyncTag, testUtils.Utils.SyncNoneTag, testUtils.Utils.SyncHardReplayTag), default=testUtils.Utils.SyncResyncTag) parser.add_argument("--kill-sig", type=str, help="kill signal[%s|%s]" % (testUtils.Utils.SigKillTag, testUtils.Utils.SigTermTag), default=testUtils.Utils.SigKillTag) parser.add_argument("--kill-count", type=int, help="nodeos instances to kill", default=-1) parser.add_argument("-v", help="verbose logging", action='store_true') -parser.add_argument("--dont-kill", help="Leave cluster running after test finishes", action='store_true') +parser.add_argument("--leave-running", help="Leave cluster running after test finishes", action='store_true') parser.add_argument("--dump-error-details", help="Upon error print etc/eosio/node_*/config.ini and var/lib/node_*/stderr.log to stdout", action='store_true') parser.add_argument("--keep-logs", help="Don't delete var/lib/node_* folders upon test completion", action='store_true') -parser.add_argument("--kill-all", help="Kill all nodeos and kleos instances", action='store_true') +parser.add_argument("--clean-run", help="Kill all nodeos and kleos instances", action='store_true') +parser.add_argument("--p2p-plugin", help="select a p2p plugin to use (either net or bnet). Defaults to net.", default="net") args = parser.parse_args() pnodes=args.p @@ -55,16 +58,18 @@ debug=args.v total_nodes = pnodes killCount=args.kill_count if args.kill_count > 0 else 1 killSignal=args.kill_sig -killEosInstances= not args.dont_kill +killEosInstances= not args.leave_running dumpErrorDetails=args.dump_error_details keepLogs=args.keep_logs -killAll=args.kill_all +killAll=args.clean_run +p2pPlugin=args.p2p_plugin seed=1 testUtils.Utils.Debug=debug testSuccessful=False - +assert (chainSyncStrategyStr == testUtils.Utils.SyncResyncTag or chainSyncStrategyStr == testUtils.Utils.SyncNoneTag or + chainSyncStrategyStr == testUtils.Utils.SyncHardReplayTag) random.seed(seed) # Use a fixed seed for repeatability. cluster=testUtils.Cluster(walletd=True) walletMgr=testUtils.WalletMgr(True) @@ -75,12 +80,14 @@ try: cluster.killall(allInstances=killAll) cluster.cleanup() + walletMgr.killall(allInstances=killAll) + walletMgr.cleanup() Print ("producing nodes: %d, topology: %s, delay between nodes launch(seconds): %d, chain sync strategy: %s" % ( pnodes, topo, delay, chainSyncStrategyStr)) Print("Stand up cluster") - if cluster.launch(pnodes, total_nodes, topo=topo, delay=delay) is False: + if cluster.launch(pnodes, total_nodes, topo=topo, delay=delay, p2pPlugin=p2pPlugin) is False: errorExit("Failed to stand up eos cluster.") Print ("Wait for Cluster stabilization") @@ -113,7 +120,6 @@ try: errorExit("Failed to import key for account %s" % (defproduceraAccount.name)) Print("Create accounts.") - #if not cluster.createAccounts(defproduceraAccount): if not cluster.createAccounts(eosioAccount): errorExit("Accounts creation failed.") @@ -121,23 +127,11 @@ try: if not cluster.waitOnClusterSync(): errorExit("Cluster sync wait failed.") - # TBD: Known issue (Issue 2043) that 'get currency0000 balance' doesn't return balance. - # Uncomment when functional - Print("Spread funds and validate") - if not cluster.spreadFundsAndValidate(10): - errorExit("Failed to spread and validate funds.") - - Print("Wait on cluster sync.") - if not cluster.waitOnClusterSync(): - errorExit("Cluster sync wait failed.") - Print("Kill %d cluster node instances." % (killCount)) if cluster.killSomeEosInstances(killCount, killSignal) is False: errorExit("Failed to kill Eos instances") Print("nodeos instances killed.") - # TBD: Known issue (Issue 2043) that 'get currency0000 balance' doesn't return balance. - # Uncomment when functional Print("Spread funds and validate") if not cluster.spreadFundsAndValidate(10): errorExit("Failed to spread and validate funds.") @@ -156,8 +150,6 @@ try: errorExit("Cluster never synchronized") Print ("Cluster synched") - # TBD: Known issue (Issue 2043) that 'get currency0000 balance' doesn't return balance. - # Uncomment when functional Print("Spread funds and validate") if not cluster.spreadFundsAndValidate(10): errorExit("Failed to spread and validate funds.") @@ -168,6 +160,10 @@ try: testSuccessful=True finally: + if testSuccessful: + Print("Test succeeded.") + else: + Print("Test failed.") if not testSuccessful and dumpErrorDetails: cluster.dumpErrorDetails() walletMgr.dumpErrorDetails() diff --git a/tests/testUtils.py b/tests/testUtils.py index 7e29fcd1581e19a9d2bdf4a53aef7e9c336afb50..6eb6c2ca16ca46de819fc10d4caec6bca9cd5457 100755 --- a/tests/testUtils.py +++ b/tests/testUtils.py @@ -89,7 +89,7 @@ class Utils: def checkOutput(cmd): assert(isinstance(cmd, list)) popen=subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (output,error)=popen.communicate(); + (output,error)=popen.communicate() if popen.returncode != 0: raise subprocess.CalledProcessError(returncode=popen.returncode, cmd=cmd, output=error) return output.decode("utf-8") @@ -305,9 +305,9 @@ class Node(object): # pylint: disable=too-many-branches def getBlock(self, blockNum, retry=True, silentErrors=False): """Given a blockId will return block details.""" - assert(isinstance(blockNum, str)) + assert(isinstance(blockNum, int)) if not self.enableMongo: - cmd="%s %s get block %s" % (Utils.EosClientPath, self.endpointArgs, blockNum) + cmd="%s %s get block %d" % (Utils.EosClientPath, self.endpointArgs, blockNum) if Utils.Debug: Utils.Print("cmd: %s" % (cmd)) try: trans=Utils.runCmdReturnJson(cmd) @@ -320,7 +320,7 @@ class Node(object): else: for _ in range(2): cmd="%s %s" % (Utils.MongoPath, self.mongoEndpointArgs) - subcommand='db.Blocks.findOne( { "block_num": %s } )' % (blockNum) + subcommand='db.Blocks.findOne( { "block_num": %d } )' % (blockNum) if Utils.Debug: Utils.Print("cmd: echo '%s' | %s" % (subcommand, cmd)) try: trans=Node.runMongoCmdReturnJson(cmd.split(), subcommand) @@ -361,21 +361,56 @@ class Node(object): return None - def doesNodeHaveBlockNum(self, blockNum): + # def doesNodeHaveBlockNum(self, blockNum): + # """Does node have head_block_num >= blockNum""" + # assert isinstance(blockNum, int) + # assert (blockNum > 0) + + # info=self.getInfo(silentErrors=True) + # assert(info) + # head_block_num=0 + # try: + # head_block_num=int(info["head_block_num"]) + # except (TypeError, KeyError) as _: + # Utils.Print("Failure in get info parsing. %s" % (info)) + # raise + + # return True if blockNum <= head_block_num else False + + def isBlockPresent(self, blockNum): + """Does node have head_block_num >= blockNum""" assert isinstance(blockNum, int) assert (blockNum > 0) info=self.getInfo(silentErrors=True) assert(info) - last_irreversible_block_num=0 + node_block_num=0 try: - last_irreversible_block_num=int(info["last_irreversible_block_num"]) + node_block_num=int(info["head_block_num"]) except (TypeError, KeyError) as _: Utils.Print("Failure in get info parsing. %s" % (info)) raise - return True if blockNum <= last_irreversible_block_num else True + return True if blockNum <= node_block_num else False + + def isBlockFinalized(self, blockNum): + """Is blockNum finalized""" + assert(blockNum) + assert isinstance(blockNum, int) + assert (blockNum > 0) + info=self.getInfo(silentErrors=True) + assert(info) + node_block_num=0 + try: + node_block_num=int(info["last_irreversible_block_num"]) + except (TypeError, KeyError) as _: + Utils.Print("Failure in get info parsing. %s" % (info)) + raise + + return True if blockNum <= node_block_num else False + + # pylint: disable=too-many-branches def getTransaction(self, transId, retry=True, silentErrors=False): if not self.enableMongo: @@ -418,7 +453,7 @@ class Node(object): assert(transId) assert(isinstance(transId, str)) assert(blockId) - assert(isinstance(blockId, str)) + assert(isinstance(blockId, int)) block=self.getBlock(blockId) transactions=None @@ -441,7 +476,7 @@ class Node(object): return False def getBlockIdByTransId(self, transId): - """Given a transaction Id (string), will return block id (string) containing the transaction""" + """Given a transaction Id (string), will return block id (int) containing the transaction""" assert(transId) assert(isinstance(transId, str)) trans=self.getTransaction(transId) @@ -463,17 +498,28 @@ class Node(object): Utils.Print("Info parsing failed. %s" % (headBlockNum)) for blockNum in range(refBlockNum, headBlockNum+1): - if self.isTransInBlock(str(transId), str(blockNum)): - return str(blockNum) + if self.isTransInBlock(str(transId), blockNum): + return blockNum return None - def doesNodeHaveTransId(self, transId): + def isTransInAnyBlock(self, transId): + """Check if transaction (transId) is in a block.""" + assert(transId) + assert(isinstance(transId, str)) + blockId=self.getBlockIdByTransId(transId) + return True if blockId else False + + def isTransFinalized(self, transId): """Check if transaction (transId) has been finalized.""" assert(transId) assert(isinstance(transId, str)) blockId=self.getBlockIdByTransId(transId) - return True if blockId else None + if not blockId: + return False + + assert(isinstance(blockId, int)) + return self.isBlockFinalized(blockId) # Disabling MongodDB funbction # def getTransByBlockId(self, blockId, retry=True, silentErrors=False): @@ -560,11 +606,11 @@ class Node(object): return None if stakedDeposit > 0: - self.waitForTransIdOnNode(transId) # seems like account creation needs to be finlized before transfer can happen + self.waitForTransInBlock(transId) # seems like account creation needs to be finalized before transfer can happen trans = self.transferFunds(creatorAccount, account, Node.currencyIntToStr(stakedDeposit, CORE_SYMBOL), "init") transId=Node.getTransId(trans) - if waitForTransBlock and not self.waitForTransIdOnNode(transId): + if waitForTransBlock and not self.waitForTransInBlock(transId): return None return trans @@ -587,11 +633,11 @@ class Node(object): return None if stakedDeposit > 0: - self.waitForTransIdOnNode(transId) # seems like account creation needs to be finlized before transfer can happen + self.waitForTransInBlock(transId) # seems like account creation needs to be finlized before transfer can happen trans = self.transferFunds(creatorAccount, account, "%0.04f %s" % (stakedDeposit/10000, CORE_SYMBOL), "init") transId=Node.getTransId(trans) - if waitForTransBlock and not self.waitForTransIdOnNode(transId): + if waitForTransBlock and not self.waitForTransInBlock(transId): return None return trans @@ -702,13 +748,16 @@ class Node(object): return None - def waitForBlockNumOnNode(self, blockNum, timeout=None): - lam = lambda: self.doesNodeHaveBlockNum(blockNum) + def waitForTransInBlock(self, transId, timeout=None): + """Wait for trans id to be finalized.""" + lam = lambda: self.isTransInAnyBlock(transId) ret=Utils.waitForBool(lam, timeout) return ret - def waitForTransIdOnNode(self, transId, timeout=None): - lam = lambda: self.doesNodeHaveTransId(transId) + def waitForTransFinalization(self, transId, timeout=None): + """Wait for trans id to be finalized.""" + assert(isinstance(transId, str)) + lam = lambda: self.isTransFinalized(transId) ret=Utils.waitForBool(lam, timeout) return ret @@ -745,7 +794,7 @@ class Node(object): assert(trans) transId=Node.getTransId(trans) - if waitForTransBlock and not self.waitForTransIdOnNode(transId): + if waitForTransBlock and not self.waitForTransInBlock(transId): return None return trans @@ -940,7 +989,7 @@ class Node(object): Node.validateTransaction(trans) transId=Node.getTransId(trans) - if waitForTransBlock and not self.waitForTransIdOnNode(transId): + if waitForTransBlock and not self.waitForTransInBlock(transId): return None return trans @@ -999,7 +1048,7 @@ class Node(object): return None transId=Node.getTransId(trans) - if waitForTransBlock and not self.waitForTransIdOnNode(transId): + if waitForTransBlock and not self.waitForTransInBlock(transId): return None return trans @@ -1086,12 +1135,30 @@ class Node(object): return True # TBD: make nodeId an internal property - def relaunch(self, nodeId, chainArg, timeout=Utils.systemWaitTimeout): + # pylint: disable=too-many-locals + def relaunch(self, nodeId, chainArg, newChain=False, timeout=Utils.systemWaitTimeout): assert(self.pid is None) assert(self.killed) if Utils.Debug: Utils.Print("Launching node process, Id: %d" % (nodeId)) + + cmdArr=[] + myCmd=self.cmd + if not newChain: + skip=False + for i in self.cmd.split(): + Utils.Print("\"%s\"" % (i)) + if skip: + skip=False + continue + if "--genesis-json" == i or "--genesis-timestamp" == i: + skip=True + continue + + cmdArr.append(i) + myCmd=" ".join(cmdArr) + dataDir="var/lib/node_%02d" % (nodeId) dt = datetime.datetime.now() dateStr="%d_%02d_%02d_%02d_%02d_%02d" % ( @@ -1099,7 +1166,8 @@ class Node(object): stdoutFile="%s/stdout.%s.txt" % (dataDir, dateStr) stderrFile="%s/stderr.%s.txt" % (dataDir, dateStr) with open(stdoutFile, 'w') as sout, open(stderrFile, 'w') as serr: - cmd=self.cmd + ("" if chainArg is None else (" " + chainArg)) + #cmd=self.cmd + ("" if chainArg is None else (" " + chainArg)) + cmd=myCmd + ("" if chainArg is None else (" " + chainArg)) Utils.Print("cmd: %s" % (cmd)) popen=subprocess.Popen(cmd.split(), stdout=sout, stderr=serr) self.pid=popen.pid @@ -1288,15 +1356,10 @@ class WalletMgr(object): with open(WalletMgr.__walletLogFile, "r") as f: shutil.copyfileobj(f, sys.stdout) - # @staticmethod - # def killall(): - # cmd="pkill -9 %s" % (Utils.EosWalletName) - # if Utils.Debug: Utils.Print("cmd: %s" % (cmd)) - # subprocess.call(cmd.split()) - def killall(self, allInstances=False): """Kill keos instances. allInstances will kill all keos instances running on the system.""" if self.__walletPid: + Utils.Print("Killing wallet manager process %d:" % (self.__walletPid)) os.kill(self.__walletPid, signal.SIGKILL) if allInstances: @@ -1384,7 +1447,7 @@ class Cluster(object): # pylint: disable=too-many-return-statements # pylint: disable=too-many-branches # pylint: disable=too-many-statements - def launch(self, pnodes=1, totalNodes=1, prodCount=1, topo="mesh", delay=1, onlyBios=False, dontKill=False, dontBootstrap=False): + def launch(self, pnodes=1, totalNodes=1, prodCount=1, topo="mesh", p2pPlugin="net", delay=1, onlyBios=False, dontKill=False, dontBootstrap=False): """Launch cluster. pnodes: producer nodes count totalNodes: producer + non-producer nodes count @@ -1400,13 +1463,13 @@ class Cluster(object): if len(self.nodes) > 0: raise RuntimeError("Cluster already running.") - cmd="%s -p %s -n %s -s %s -d %s -i %s -f --p2p-plugin bnet" % ( - Utils.EosLauncherPath, pnodes, totalNodes, topo, delay, datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3]) + cmd="%s -p %s -n %s -s %s -d %s -i %s -f --p2p-plugin %s" % ( + Utils.EosLauncherPath, pnodes, totalNodes, topo, delay, datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3], p2pPlugin) cmdArr=cmd.split() if self.staging: cmdArr.append("--nogen") - nodeosArgs="--max-transaction-time 5000 --filter-on *" + nodeosArgs="--max-transaction-time 5000 --filter-on * --p2p-max-nodes-per-host %d" % (totalNodes) if not self.walletd: nodeosArgs += " --plugin eosio::wallet_api_plugin" if self.enableMongo: @@ -1533,13 +1596,14 @@ class Cluster(object): self.nodes=nodes return True - # manually set nodes, alternative to explicit launch def setNodes(self, nodes): + """manually set nodes, alternative to explicit launch""" self.nodes=nodes - # If a last transaction exists wait for it on root node, then collect its head block number. - # Wait on this block number on each cluster node def waitOnClusterSync(self, timeout=None): + """Get head block on node 0, then ensure the block is present on every cluster node.""" + assert(self.nodes) + assert(len(self.nodes) > 0) targetHeadBlockNum=self.nodes[0].getHeadBlockNum() #get root nodes head block num if Utils.Debug: Utils.Print("Head block number on root node: %d" % (targetHeadBlockNum)) if targetHeadBlockNum == -1: @@ -1547,16 +1611,23 @@ class Cluster(object): return self.waitOnClusterBlockNumSync(targetHeadBlockNum, timeout) - def waitOnClusterBlockNumSync(self, targetHeadBlockNum, timeout=None): + def waitOnClusterBlockNumSync(self, targetBlockNum, timeout=None): + """Wait for all nodes to have targetBlockNum finalized.""" + assert(self.nodes) - def doNodesHaveBlockNum(nodes, targetHeadBlockNum): + def doNodesHaveBlockNum(nodes, targetBlockNum): for node in nodes: - if (not node.killed) and (not node.doesNodeHaveBlockNum(targetHeadBlockNum)): + try: + if (not node.killed) and (not node.isBlockPresent(targetBlockNum)): + #if (not node.killed) and (not node.isBlockFinalized(targetBlockNum)): + return False + except (TypeError) as _: + # This can happen if client connects before server is listening return False return True - lam = lambda: doNodesHaveBlockNum(self.nodes, targetHeadBlockNum) + lam = lambda: doNodesHaveBlockNum(self.nodes, targetBlockNum) ret=Utils.waitForBool(lam, timeout) return ret @@ -1692,8 +1763,8 @@ class Cluster(object): #Utils.Print("nextEosIdx: %d, count: %d" % (nextEosIdx, count)) node=self.nodes[nextEosIdx] - if Utils.Debug: Utils.Print("Wait for trasaction id %s on node port %d" % (transId, node.port)) - if node.waitForTransIdOnNode(transId) is False: + if Utils.Debug: Utils.Print("Wait for transaction id %s on node port %d" % (transId, node.port)) + if node.waitForTransInBlock(transId) is False: Utils.Print("ERROR: Selected node never received transaction id %s" % (transId)) return False @@ -1713,8 +1784,8 @@ class Cluster(object): # As an extra step wait for last transaction on the root node node=self.nodes[0] - if Utils.Debug: Utils.Print("Wait for trasaction id %s on node port %d" % (transId, node.port)) - if node.waitForTransIdOnNode(transId) is False: + if Utils.Debug: Utils.Print("Wait for transaction id %s on node port %d" % (transId, node.port)) + if node.waitForTransInBlock(transId) is False: Utils.Print("ERROR: Selected node never received transaction id %s" % (transId)) return False @@ -1946,7 +2017,7 @@ class Cluster(object): accounts.append(initx) transId=Node.getTransId(trans) - biosNode.waitForTransIdOnNode(transId) + biosNode.waitForTransInBlock(transId) Utils.Print("Validating system accounts within bootstrap") biosNode.validateAccounts(accounts) @@ -1993,7 +2064,7 @@ class Cluster(object): trans=trans[1] transId=Node.getTransId(trans) - if not biosNode.waitForTransIdOnNode(transId): + if not biosNode.waitForTransInBlock(transId): return False # wait for block production handover (essentially a block produced by anyone but eosio). @@ -2033,7 +2104,7 @@ class Cluster(object): Node.validateTransaction(trans) transId=Node.getTransId(trans) - biosNode.waitForTransIdOnNode(transId) + biosNode.waitForTransInBlock(transId) contract="eosio.token" contractDir="contracts/%s" % (contract) @@ -2058,7 +2129,7 @@ class Cluster(object): Node.validateTransaction(trans[1]) transId=Node.getTransId(trans[1]) - biosNode.waitForTransIdOnNode(transId) + biosNode.waitForTransInBlock(transId) contract=eosioTokenAccount.name Utils.Print("push issue action to %s contract" % (contract)) @@ -2073,7 +2144,8 @@ class Cluster(object): Node.validateTransaction(trans[1]) Utils.Print("Wait for issue action transaction to become finalized.") transId=Node.getTransId(trans[1]) - biosNode.waitForTransIdOnNode(transId) + # biosNode.waitForTransInBlock(transId) + biosNode.waitForTransFinalization(transId) expectedAmount="1000000000.0000 {0}".format(CORE_SYMBOL) Utils.Print("Verify eosio issue, Expected: %s" % (expectedAmount)) @@ -2112,7 +2184,7 @@ class Cluster(object): Utils.Print("Wait for last transfer transaction to become finalized.") transId=Node.getTransId(trans[1]) - if not biosNode.waitForTransIdOnNode(transId): + if not biosNode.waitForTransInBlock(transId): return False Utils.Print("Cluster bootstrap done.") @@ -2152,7 +2224,7 @@ class Cluster(object): if Utils.Debug: Utils.Print("pgrep output: \"%s\"" % psOut) for i in range(0, totalNodes): - pattern=r"[\n]?(\d+) (.* --data-dir var/lib/node_%02d)" % (i) + pattern=r"[\n]?(\d+) (.* --data-dir var/lib/node_%02d .*)\n" % (i) m=re.search(pattern, psOut, re.MULTILINE) if m is None: Utils.Print("ERROR: Failed to find %s pid. Pattern %s" % (Utils.EosServerName, pattern)) @@ -2187,9 +2259,10 @@ class Cluster(object): chainArg=self.__chainSyncStrategy.arg + newChain= False if self.__chainSyncStrategy.name == Utils.SyncHardReplayTag else True for i in range(0, len(self.nodes)): node=self.nodes[i] - if node.killed and not node.relaunch(i, chainArg): + if node.killed and not node.relaunch(i, chainArg, newChain=newChain): return False return True @@ -2288,7 +2361,7 @@ class Cluster(object): if waitForTransBlock and transId is not None: node=self.nodes[0] if Utils.Debug: Utils.Print("Wait for transaction id %s on server port %d." % ( transId, node.port)) - if node.waitForTransIdOnNode(transId) is False: + if node.waitForTransInBlock(transId) is False: Utils.Print("ERROR: Failed waiting for transaction id %s on server port %d." % ( transId, node.port)) return False diff --git a/tests/validate-dirty-db.py b/tests/validate-dirty-db.py index 6cef6b888799b4fae7c5241f60f0afa2ccfa5de2..8d882ea438eb4fb2d68749d18e082700bb71b9c6 100755 --- a/tests/validate-dirty-db.py +++ b/tests/validate-dirty-db.py @@ -5,8 +5,6 @@ import testUtils import argparse import random import subprocess -import time -import os import signal ############################################################### @@ -22,12 +20,13 @@ def errorExit(msg="", errorCode=1): parser = argparse.ArgumentParser() parser.add_argument("-v", help="verbose logging", action='store_true') -parser.add_argument("--dont-kill", help="Leave cluster running after test finishes", action='store_true') +parser.add_argument("--leave-running", help="Leave cluster running after test finishes", action='store_true') parser.add_argument("--dump-error-details", help="Upon error print etc/eosio/node_*/config.ini and var/lib/node_*/stderr.log to stdout", action='store_true') parser.add_argument("--keep-logs", help="Don't delete var/lib/node_* folders upon test completion", action='store_true') +parser.add_argument("--clean-run", help="Kill all nodeos and kleos instances", action='store_true') args = parser.parse_args() debug=args.v @@ -39,21 +38,42 @@ total_nodes = pnodes killCount=1 killSignal=testUtils.Utils.SigKillTag -killEosInstances= not args.dont_kill +killEosInstances= not args.leave_running dumpErrorDetails=args.dump_error_details keepLogs=args.keep_logs +killAll=args.clean_run seed=1 testUtils.Utils.Debug=debug testSuccessful=False +def runNodeosAndGetOutput(myNodeId, myTimeout=3): + """Startup nodeos, wait for timeout (before forced shutdown) and collect output. Stdout, stderr and return code are returned in a dictionary.""" + Print("Launching nodeos process id: %d" % (myNodeId)) + cmd="programs/nodeos/nodeos --config-dir etc/eosio/node_bios --data-dir var/lib/node_bios" + Print("cmd: %s" % (cmd)) + proc=subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + output={} + try: + outs,errs = proc.communicate(timeout=myTimeout) + output["stdout"] = outs.decode("utf-8") + output["stderr"] = errs.decode("utf-8") + output["returncode"] = proc.returncode + except (subprocess.TimeoutExpired) as _: + Print("ERROR: Nodeos is running beyond the defined wait time. Hard killing nodeos instance.") + proc.send_signal(signal.SIGKILL) + return (False, None) + + return (True, output) + random.seed(seed) # Use a fixed seed for repeatability. cluster=testUtils.Cluster(walletd=True) try: cluster.setChainStrategy(chainSyncStrategyStr) - cluster.killall() + cluster.killall(allInstances=killAll) cluster.cleanup() Print ("producing nodes: %d, topology: %s, delay between nodes launch(seconds): %d, chain sync strategy: %s" % ( @@ -68,28 +88,7 @@ try: errorExit("Cluster in bad state, received None node") Print("Kill cluster nodes.") - cluster.killall() - - def runNodeosAndGetOutput(nodeId, timeout=3): - """Startup nodeos, wait for timeout (before forced shutdown) and collect output. Stdout, stderr and return code are returned in a dictionary.""" - Print("Launching nodeos process id: %d" % (nodeId)) - dataDir="var/lib/node_%02d" % (nodeId) - cmd="programs/nodeos/nodeos --config-dir etc/eosio/node_bios --data-dir var/lib/node_bios" - Print("cmd: %s" % (cmd)) - proc=subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - output={} - try: - outs,errs = proc.communicate(timeout=timeout) - output["stdout"] = outs.decode("utf-8") - output["stderr"] = errs.decode("utf-8") - output["returncode"] = proc.returncode - except (subprocess.TimeoutExpired) as _: - Print("ERROR: Nodeos is running beyond the defined wait time. Hard killing nodeos instance.") - proc.send_signal(signal.SIGKILL) - return (False, None) - - return (True, output) + cluster.killall(allInstances=killAll) Print("Restart nodeos repeatedly to ensure dirty database flag sticks.") nodeId=0 @@ -98,12 +97,15 @@ try: for i in range(0,3): Print("Attempt %d." % (i)) ret = runNodeosAndGetOutput(nodeId, timeout) + assert(ret) + assert(isinstance(ret, tuple)) if not ret or not ret[0]: exit(1) - #Print(ret) - - stderr=ret[1]["stderr"] + assert(ret[1]) + assert(isinstance(ret[1], dict)) + # pylint: disable=unsubscriptable-object + stderr= ret[1]["stderr"] retCode=ret[1]["returncode"] assert(retCode == 2) assert("database dirty flag set" in stderr) @@ -121,7 +123,7 @@ finally: if killEosInstances: Print("Shut down the cluster.") - cluster.killall() + cluster.killall(allInstances=killAll) if testSuccessful and not keepLogs: Print("Cleanup cluster data.") cluster.cleanup()