Merge pull request #5476 from EOSIO/launcher-tests

Launcher tests

plugins/history_plugin/history_plugin.cpp

@@ -313,7 +313,7 @@ namespace eosio {
          if( options.count( "filter-on" )) {
             auto fo = options.at( "filter-on" ).as<vector<string>>();
             for( auto& s : fo ) {
-               if( s == "*" ) {
+               if( s == "*" || s == "\"*\"" ) {
                   my->bypass_filter = true;
                   wlog( "--filter-on * enabled. This can fill shared_mem, causing nodeos to stop." );
                   break;

programs/eosio-launcher/main.cpp

@@ -17,6 +17,7 @@
 #pragma GCC diagnostic ignored "-Wunused-result"
 #include <boost/process/child.hpp>
 #pragma GCC diagnostic pop
+#include <boost/process/env.hpp>
 #include <boost/process/system.hpp>
 #include <boost/process/io.hpp>
 #include <boost/lexical_cast.hpp>
@@ -162,7 +163,7 @@ public:
      return base_http_port - 100;
   }
 
-  bool is_local( ) {
+  bool is_local( ) const {
     return local_id.contains( host_name );
   }
 
@@ -457,6 +458,7 @@ struct launcher_def {
   void make_custom ();
   void write_dot_file ();
   void format_ssh (const string &cmd, const string &host_name, string &ssh_cmd_line);
+  void do_command(const host_def& host, const string& name, vector<pair<string, string>> env_pairs, const string& cmd);
   bool do_ssh (const string &cmd, const string &host_name);
   void prep_remote_config_dir (eosd_def &node, host_def *host);
   void launch (eosd_def &node, string &gts);
@@ -1643,6 +1645,30 @@ launcher_def::get_nodes(const string& node_number_list) {
    return node_list;
 }
 
+void
+launcher_def::do_command(const host_def& host, const string& name,
+                         vector<pair<string, string>> env_pairs, const string& cmd) {
+   if (!host.is_local()) {
+      string rcmd = "cd " + host.eosio_home + "; ";
+      for (auto& env_pair : env_pairs) {
+         rcmd += "export " + env_pair.first + "=" + env_pair.second + "; ";
+      }
+      rcmd += cmd;
+      if (!do_ssh(rcmd, host.host_name)) {
+         cerr << "Remote command failed for " << name << endl;
+         exit (-1);
+      }
+   }
+   else {
+      bp::environment e;
+      for (auto& env_pair : env_pairs) {
+         e.emplace(env_pair.first, env_pair.second);
+      }
+      bp::child c(cmd, e);
+      c.wait();
+   }
+}
+
 void
 launcher_def::bounce (const string& node_numbers) {
    auto node_list = get_nodes(node_numbers);
@@ -1650,15 +1676,9 @@ launcher_def::bounce (const string& node_numbers) {
       const host_def& host = node_pair.first;
       const eosd_def& node = node_pair.second;
       string node_num = node.name.substr( node.name.length() - 2 );
-      string cmd = "cd " + host.eosio_home + "; "
-                 + "export EOSIO_HOME=" + host.eosio_home + string("; ")
-                 + "export EOSIO_NODE=" + node_num + "; "
-                 + "./scripts/eosio-tn_bounce.sh " + eosd_extra_args;
       cout << "Bouncing " << node.name << endl;
-      if (!do_ssh(cmd, host.host_name)) {
-         cerr << "Unable to bounce " << node.name << endl;
-         exit (-1);
-      }
+      string cmd = "./scripts/eosio-tn_bounce.sh " + eosd_extra_args;
+      do_command(host, node.name, { { "EOSIO_HOME", host.eosio_home }, { "EOSIO_NODE", node_num } }, cmd);
    }
 }
 
@@ -1669,16 +1689,11 @@ launcher_def::down (const string& node_numbers) {
       const host_def& host = node_pair.first;
       const eosd_def& node = node_pair.second;
       string node_num = node.name.substr( node.name.length() - 2 );
-      string cmd = "cd " + host.eosio_home + "; "
-                 + "export EOSIO_HOME=" + host.eosio_home + "; "
-                 + "export EOSIO_NODE=" + node_num + "; "
-         + "export EOSIO_TN_RESTART_CONFIG_DIR=" + node.config_dir_name + "; "
-                 + "./scripts/eosio-tn_down.sh";
       cout << "Taking down " << node.name << endl;
-      if (!do_ssh(cmd, host.host_name)) {
-         cerr << "Unable to down " << node.name << endl;
-         exit (-1);
-      }
+      string cmd = "./scripts/eosio-tn_down.sh ";
+      do_command(host, node.name,
+                 { { "EOSIO_HOME", host.eosio_home }, { "EOSIO_NODE", node_num }, { "EOSIO_TN_RESTART_CONFIG_DIR", node.config_dir_name } },
+                 cmd);
    }
 }
 
@@ -1689,13 +1704,8 @@ launcher_def::roll (const string& host_names) {
    for (string host_name: hosts) {
       cout << "Rolling " << host_name << endl;
       auto host = find_host_by_name_or_address(host_name);
-      string cmd = "cd " + host->eosio_home + "; "
-                 + "export EOSIO_HOME=" + host->eosio_home + "; "
-                 + "./scripts/eosio-tn_roll.sh";
-      if (!do_ssh(cmd, host_name)) {
-         cerr << "Unable to roll " << host << endl;
-         exit (-1);
-      }
+      string cmd = "./scripts/eosio-tn_roll.sh ";
+      do_command(*host, host_name, { { "EOSIO_HOME", host->eosio_home } }, cmd);
    }
 }
 

scripts/eosio-tn_bounce.sh

@@ -41,4 +41,4 @@ else
 fi
 
 bash $EOSIO_HOME/scripts/eosio-tn_down.sh
-bash $EOSIO_HOME/scripts/eosio-tn_up.sh $*
+bash $EOSIO_HOME/scripts/eosio-tn_up.sh "$*"

scripts/eosio-tn_roll.sh

@@ -82,10 +82,10 @@ cp $SDIR/$RD/$prog $RD/$prog
 
 if [ $DD = "all" ]; then
     for EOSIO_RESTART_DATA_DIR in `ls -d var/lib/node_??`; do
-        bash $EOSIO_HOME/scripts/eosio-tn_up.sh $*
+        bash $EOSIO_HOME/scripts/eosio-tn_up.sh "$*"
     done
 else
-    bash $EOSIO_HOME/scripts/eosio-tn_up.sh $*
+    bash $EOSIO_HOME/scripts/eosio-tn_up.sh "$*"
 fi
 unset EOSIO_RESTART_DATA_DIR
 

scripts/eosio-tn_up.sh

@@ -9,6 +9,8 @@ connected="0"
 rundir=programs/nodeos
 prog=nodeos
 
+# Quote any args that are "*", so they are not expanded
+qargs=`echo "$*" | sed -e 's/ \* / "*" /' -e 's/ \*$/ "*"/'`
 
 if [ "$PWD" != "$EOSIO_HOME" ]; then
     echo $0 must only be run from $EOSIO_HOME
@@ -33,8 +35,8 @@ rm $datadir/stderr.txt
 ln -s $log $datadir/stderr.txt
 
 relaunch() {
-    echo "$rundir/$prog $* --data-dir $datadir --config-dir etc/eosio/node_$EOSIO_NODE > $datadir/stdout.txt  2>> $datadir/$log "
-    nohup $rundir/$prog $* --data-dir $datadir --config-dir etc/eosio/node_$EOSIO_NODE > $datadir/stdout.txt  2>> $datadir/$log &
+    echo "$rundir/$prog $qargs $* --data-dir $datadir --config-dir etc/eosio/node_$EOSIO_NODE > $datadir/stdout.txt  2>> $datadir/$log "
+    nohup $rundir/$prog $qargs $* --data-dir $datadir --config-dir etc/eosio/node_$EOSIO_NODE > $datadir/stdout.txt  2>> $datadir/$log &
     pid=$!
     echo pid = $pid
     echo $pid > $datadir/$prog.pid
@@ -56,7 +58,7 @@ relaunch() {
 
 if [ -z "$EOSIO_LEVEL" ]; then
     echo starting with no modifiers
-    relaunch $*
+    relaunch
     if [ "$connected" -eq 0 ]; then
         EOSIO_LEVEL=replay
     else
@@ -66,7 +68,7 @@ fi
 
 if [ "$EOSIO_LEVEL" == replay ]; then
     echo starting with replay
-    relaunch $* --hard-replay-blockchain
+    relaunch --hard-replay-blockchain
     if [  "$connected" -eq 0 ]; then
         EOSIO_LEVEL=resync
     else
@@ -75,5 +77,5 @@ if [ "$EOSIO_LEVEL" == replay ]; then
 fi
 if [ "$EOSIO_LEVEL" == resync ]; then
     echo starting with delete-all-blocks
-    relaunch $* --delete-all-blocks
+    relaunch --delete-all-blocks
 fi

tests/CMakeLists.txt

@@ -40,6 +40,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_under_min_avail_ram.py ${CMAKE
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_voting_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_voting_test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/consensus-validation-malicious-producers.py ${CMAKE_CURRENT_BINARY_DIR}/consensus-validation-malicious-producers.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/validate-dirty-db.py ${CMAKE_CURRENT_BINARY_DIR}/validate-dirty-db.py COPYONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/launcher_test.py ${CMAKE_CURRENT_BINARY_DIR}/launcher_test.py COPYONLY)
 
 #To run plugin_test with all log from blockchain displayed, put --verbose after --, i.e. plugin_test -- --verbose
 add_test(NAME plugin_test COMMAND plugin_test --report_level=detailed --color_output)
@@ -59,6 +60,7 @@ add_test(NAME restart-scenarios-test-hard_replay COMMAND tests/restart-scenarios
 add_test(NAME restart-scenarios-test-none COMMAND tests/restart-scenarios-test.py -c none --kill-sig term -p4 -v --clean-run --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 # TODO: add_test(NAME consensus-validation-malicious-producers COMMAND tests/consensus-validation-malicious-producers.py -w 80 --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 add_test(NAME validate_dirty_db_test COMMAND tests/validate-dirty-db.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME launcher_test COMMAND tests/launcher_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 
 # Long running tests
 add_test(NAME nodeos_sanity_lr_test COMMAND tests/nodeos_run_test.py -v --sanity-test --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR})

tests/Cluster.py

@@ -30,6 +30,7 @@ class Cluster(object):
     __localHost="localhost"
     __BiosHost="localhost"
     __BiosPort=8788
+    __LauncherCmdArr=[]
 
     # pylint: disable=too-many-arguments
     # walletd [True|False] Is keosd running. If not load the wallet plugin
@@ -97,14 +98,19 @@ class Cluster(object):
     # pylint: disable=too-many-branches
     # pylint: disable=too-many-statements
     def launch(self, pnodes=1, totalNodes=1, prodCount=1, topo="mesh", p2pPlugin="net", delay=1, onlyBios=False, dontKill=False
-               , dontBootstrap=False, totalProducers=None, extraNodeosArgs=None):
+               , dontBootstrap=False, totalProducers=None, extraNodeosArgs=None, useBiosBootFile=True):
         """Launch cluster.
         pnodes: producer nodes count
         totalNodes: producer + non-producer nodes count
-        prodCount: producers per prodcuer node count
+        prodCount: producers per producer node count
         topo: cluster topology (as defined by launcher)
         delay: delay between individual nodes launch (as defined by launcher)
           delay 0 exposes a bootstrap bug where producer handover may have a large gap confusing nodes and bringing system to a halt.
+        dontBootstrap: When true, don't do any bootstrapping at all.
+        onlyBios: When true, only loads the bios contract (and not more full bootstrapping).
+        useBiosBootFile: determines which of two bootstrap methods is used (when both dontBootstrap and onlyBios are false).
+          The default value of true uses the bios_boot.sh file generated by the launcher.
+          A value of false uses manual bootstrapping in this script, which does not do things like stake votes for producers.
         """
         if not self.localCluster:
             Utils.Print("WARNING: Cluster not local, not launching %s." % (Utils.EosServerName))
@@ -144,6 +150,7 @@ class Cluster(object):
             cmdArr.append("--nodeos")
             cmdArr.append(nodeosArgs)
 
+        Cluster.__LauncherCmdArr = cmdArr.copy()
         s=" ".join(cmdArr)
         if Utils.Debug: Utils.Print("cmd: %s" % (s))
         if 0 != subprocess.call(cmdArr):
@@ -180,10 +187,16 @@ class Cluster(object):
             return True
 
         Utils.Print("Bootstrap cluster.")
-        self.biosNode=Cluster.bootstrap(totalNodes, prodCount, Cluster.__BiosHost, Cluster.__BiosPort, dontKill, onlyBios)
-        if self.biosNode is None:
-            Utils.Print("ERROR: Bootstrap failed.")
-            return False
+        if onlyBios or not useBiosBootFile:
+            self.biosNode=Cluster.bootstrap(totalNodes, prodCount, Cluster.__BiosHost, Cluster.__BiosPort, dontKill, onlyBios)
+            if self.biosNode is None:
+                Utils.Print("ERROR: Bootstrap failed.")
+                return False
+        else:
+            self.biosNode=Cluster.bios_bootstrap(totalNodes, Cluster.__BiosHost, Cluster.__BiosPort, dontKill)
+            if self.biosNode is None:
+                Utils.Print("ERROR: Bootstrap failed.")
+                return False
 
         # validate iniX accounts can be retrieved
 
@@ -572,7 +585,7 @@ class Cluster(object):
 
         node.validateAccounts(myAccounts)
 
-    def createAccountAndVerify(self, account, creator, stakedDeposit=1000, stakeNet=100, stakeCPU=100, buyRAM=100):
+    def createAccountAndVerify(self, account, creator, stakedDeposit=1000, stakeNet=100, stakeCPU=100, buyRAM=10000):
         """create account, verify account and return transaction id"""
         assert(len(self.nodes) > 0)
         node=self.nodes[0]
@@ -593,7 +606,7 @@ class Cluster(object):
     #         return transId
     #     return None
 
-    def createInitializeAccount(self, account, creatorAccount, stakedDeposit=1000, waitForTransBlock=False, stakeNet=100, stakeCPU=100, buyRAM=100, exitOnError=False):
+    def createInitializeAccount(self, account, creatorAccount, stakedDeposit=1000, waitForTransBlock=False, stakeNet=100, stakeCPU=100, buyRAM=10000, exitOnError=False):
         assert(len(self.nodes) > 0)
         node=self.nodes[0]
         trans=node.createInitializeAccount(account, creatorAccount, stakedDeposit, waitForTransBlock, stakeNet=stakeNet, stakeCPU=stakeCPU, buyRAM=buyRAM)
@@ -683,6 +696,95 @@ class Cluster(object):
 
         return producerKeys
 
+    @staticmethod
+    def bios_bootstrap(totalNodes, biosHost, biosPort, dontKill=False):
+        """Bootstrap cluster using the bios_boot.sh script generated by eosio-launcher."""
+
+        Utils.Print("Starting cluster bootstrap.")
+        biosNode=Node(biosHost, biosPort)
+        if not biosNode.checkPulse():
+            Utils.Print("ERROR: Bios node doesn't appear to be running...")
+            return None
+
+        cmd="bash bios_boot.sh"
+        if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
+        if 0 != subprocess.call(cmd.split(), stdout=Utils.FNull):
+            if not silent: Utils.Print("Launcher failed to shut down eos cluster.")
+            return None
+
+        p = re.compile('error', re.IGNORECASE)
+        bootlog="eosio-ignition-wd/bootlog.txt"
+        with open(bootlog) as bootFile:
+            for line in bootFile:
+                if p.search(line):
+                    Utils.Print("ERROR: bios_boot.sh script resulted in errors. See %s" % (bootlog))
+                    Utils.Print(line)
+                    return None
+
+        producerKeys=Cluster.parseClusterKeys(totalNodes)
+        # should have totalNodes node plus bios node
+        if producerKeys is None or len(producerKeys) < (totalNodes+1):
+            Utils.Print("ERROR: Failed to parse private keys from cluster config files.")
+            return None
+
+        walletMgr=WalletMgr(True)
+        walletMgr.killall()
+        walletMgr.cleanup()
+
+        if not walletMgr.launch():
+            Utils.Print("ERROR: Failed to launch bootstrap wallet.")
+            return None
+        biosNode.setWalletEndpointArgs(walletMgr.walletEndpointArgs)
+
+        try:
+            ignWallet=walletMgr.create("ignition")
+            if ignWallet is None:
+                Utils.Print("ERROR: Failed to create ignition wallet.")
+                return None
+
+            eosioName="eosio"
+            eosioKeys=producerKeys[eosioName]
+            eosioAccount=Account(eosioName)
+            eosioAccount.ownerPrivateKey=eosioKeys["private"]
+            eosioAccount.ownerPublicKey=eosioKeys["public"]
+            eosioAccount.activePrivateKey=eosioKeys["private"]
+            eosioAccount.activePublicKey=eosioKeys["public"]
+            producerKeys.pop(eosioName)
+
+            if not walletMgr.importKey(eosioAccount, ignWallet):
+                Utils.Print("ERROR: Failed to import %s account keys into ignition wallet." % (eosioName))
+                return None
+
+            initialFunds="1000000.0000 {0}".format(CORE_SYMBOL)
+            Utils.Print("Transfer initial fund %s to individual accounts." % (initialFunds))
+            trans=None
+            contract="eosio.token"
+            action="transfer"
+            for name, keys in producerKeys.items():
+                data="{\"from\":\"eosio\",\"to\":\"%s\",\"quantity\":\"%s\",\"memo\":\"%s\"}" % (name, initialFunds, "init transfer")
+                opts="--permission eosio@active"
+                if name != "eosio":
+                    trans=biosNode.pushMessage(contract, action, data, opts)
+                    if trans is None or not trans[0]:
+                        Utils.Print("ERROR: Failed to transfer funds from eosio.token to %s." % (name))
+                        return None
+
+                Node.validateTransaction(trans[1])
+
+            Utils.Print("Wait for last transfer transaction to become finalized.")
+            transId=Node.getTransId(trans[1])
+            if not biosNode.waitForTransInBlock(transId):
+                Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port))
+                return None
+
+            Utils.Print("Cluster bootstrap done.")
+        finally:
+            if not dontKill:
+                walletMgr.killall()
+                walletMgr.cleanup()
+
+        return biosNode
+
     @staticmethod
     def bootstrap(totalNodes, prodCount, biosHost, biosPort, dontKill=False, onlyBios=False):
         """Create 'prodCount' init accounts and deposits 10000000000 SYS in each. If prodCount is -1 will initialize all possible producers.
@@ -1060,6 +1162,32 @@ class Cluster(object):
             except OSError as _:
                 pass
 
+    def bounce(self, nodes, silent=True):
+        """Bounces nodeos instances as indicated by parameter nodes.
+        nodes should take the form of a comma-separated list as accepted by the launcher --bounce command (e.g. '00' or '00,01')"""
+        cmdArr = Cluster.__LauncherCmdArr.copy()
+        cmdArr.append("--bounce")
+        cmdArr.append(nodes)
+        cmd=" ".join(cmdArr)
+        if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
+        if 0 != subprocess.call(cmdArr):
+            if not silent: Utils.Print("Launcher failed to bounce nodes: %s." % (nodes))
+            return False
+        return True
+
+    def down(self, nodes, silent=True):
+        """Brings down nodeos instances as indicated by parameter nodes.
+        nodes should take the form of a comma-separated list as accepted by the launcher --bounce command (e.g. '00' or '00,01')"""
+        cmdArr = Cluster.__LauncherCmdArr.copy()
+        cmdArr.append("--down")
+        cmdArr.append(nodes)
+        cmd=" ".join(cmdArr)
+        if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
+        if 0 != subprocess.call(cmdArr):
+            if not silent: Utils.Print("Launcher failed to take down nodes: %s." % (nodes))
+            return False
+        return True
+
     def isMongodDbRunning(self):
         cmd="%s %s" % (Utils.MongoPath, self.mongoEndpointArgs)
         subcommand="db.version()"

tests/Node.py

@@ -411,7 +411,7 @@ class Node(object):
 
 
     # Create & initialize account and return creation transactions. Return transaction json object
-    def createInitializeAccount(self, account, creatorAccount, stakedDeposit=1000, waitForTransBlock=False, stakeNet=100, stakeCPU=100, buyRAM=100, exitOnError=False):
+    def createInitializeAccount(self, account, creatorAccount, stakedDeposit=1000, waitForTransBlock=False, stakeNet=100, stakeCPU=100, buyRAM=10000, exitOnError=False):
         cmdDesc="system newaccount"
         cmd='%s -j %s %s %s %s --stake-net "%s %s" --stake-cpu "%s %s" --buy-ram "%s %s"' % (
             cmdDesc, creatorAccount.name, account.name, account.ownerPublicKey,

tests/launcher_test.py

+#!/usr/bin/env python3
+
+from testUtils import Utils
+from Cluster import Cluster
+from WalletMgr import WalletMgr
+from Node import Node
+from TestHelper import TestHelper
+
+import decimal
+import re
+
+###############################################################
+# nodeos_run_test
+# --dump-error-details <Upon error print etc/eosio/node_*/config.ini and var/lib/node_*/stderr.log to stdout>
+# --keep-logs <Don't delete var/lib/node_* folders upon test completion>
+###############################################################
+
+Print=Utils.Print
+errorExit=Utils.errorExit
+cmdError=Utils.cmdError
+from core_symbol import CORE_SYMBOL
+
+args = TestHelper.parse_args({"--defproducera_prvt_key","--dump-error-details","--dont-launch","--keep-logs",
+                              "-v","--leave-running","--clean-run","--p2p-plugin"})
+debug=args.v
+defproduceraPrvtKey=args.defproducera_prvt_key
+dumpErrorDetails=args.dump_error_details
+keepLogs=args.keep_logs
+dontLaunch=args.dont_launch
+dontKill=args.leave_running
+killAll=args.clean_run
+p2pPlugin=args.p2p_plugin
+
+Utils.Debug=debug
+cluster=Cluster(walletd=True, defproduceraPrvtKey=defproduceraPrvtKey)
+walletMgr=WalletMgr(True)
+testSuccessful=False
+killEosInstances=not dontKill
+killWallet=not dontKill
+
+WalletdName="keosd"
+ClientName="cleos"
+timeout = .5 * 12 * 2 + 60 # time for finalization with 1 producer + 60 seconds padding
+Utils.setIrreversibleTimeout(timeout)
+
+try:
+    TestHelper.printSystemInfo("BEGIN")
+
+    walletMgr.killall(allInstances=killAll)
+    walletMgr.cleanup()
+
+    if not dontLaunch:
+        cluster.killall(allInstances=killAll)
+        cluster.cleanup()
+        Print("Stand up cluster")
+        if cluster.launch(pnodes=4, dontKill=dontKill, p2pPlugin=p2pPlugin) is False:
+            cmdError("launcher")
+            errorExit("Failed to stand up eos cluster.")
+    else:
+        cluster.initializeNodes(defproduceraPrvtKey=defproduceraPrvtKey)
+        killEosInstances=False
+
+    Print("Validating system accounts after bootstrap")
+    cluster.validateAccounts(None)
+
+    accounts=Cluster.createAccountKeys(3)
+    if accounts is None:
+        errorExit("FAILURE - create keys")
+    testeraAccount=accounts[0]
+    testeraAccount.name="testera11111"
+    currencyAccount=accounts[1]
+    currencyAccount.name="currency1111"
+    exchangeAccount=accounts[2]
+    exchangeAccount.name="exchange1111"
+
+    PRV_KEY1=testeraAccount.ownerPrivateKey
+    PUB_KEY1=testeraAccount.ownerPublicKey
+    PRV_KEY2=currencyAccount.ownerPrivateKey
+    PUB_KEY2=currencyAccount.ownerPublicKey
+    PRV_KEY3=exchangeAccount.activePrivateKey
+    PUB_KEY3=exchangeAccount.activePublicKey
+
+    testeraAccount.activePrivateKey=currencyAccount.activePrivateKey=PRV_KEY3
+    testeraAccount.activePublicKey=currencyAccount.activePublicKey=PUB_KEY3
+
+    exchangeAccount.ownerPrivateKey=PRV_KEY2
+    exchangeAccount.ownerPublicKey=PUB_KEY2
+
+    Print("Stand up %s" % (WalletdName))
+    walletMgr.killall(allInstances=killAll)
+    walletMgr.cleanup()
+    if walletMgr.launch() is False:
+        cmdError("%s" % (WalletdName))
+        errorExit("Failed to stand up eos walletd.")
+
+    testWalletName="test"
+    Print("Creating wallet \"%s\"." % (testWalletName))
+    testWallet=walletMgr.create(testWalletName, [cluster.eosioAccount,cluster.defproduceraAccount])
+
+    Print("Wallet \"%s\" password=%s." % (testWalletName, testWallet.password.encode("utf-8")))
+
+    for account in accounts:
+        Print("Importing keys for account %s into wallet %s." % (account.name, testWallet.name))
+        if not walletMgr.importKey(account, testWallet):
+            cmdError("%s wallet import" % (ClientName))
+            errorExit("Failed to import key for account %s" % (account.name))
+
+    defproduceraWalletName="defproducera"
+    Print("Creating wallet \"%s\"." % (defproduceraWalletName))
+    defproduceraWallet=walletMgr.create(defproduceraWalletName)
+
+    Print("Wallet \"%s\" password=%s." % (defproduceraWalletName, defproduceraWallet.password.encode("utf-8")))
+
+    defproduceraAccount=cluster.defproduceraAccount
+
+    Print("Importing keys for account %s into wallet %s." % (defproduceraAccount.name, defproduceraWallet.name))
+    if not walletMgr.importKey(defproduceraAccount, defproduceraWallet):
+        cmdError("%s wallet import" % (ClientName))
+        errorExit("Failed to import key for account %s" % (defproduceraAccount.name))
+
+    node=cluster.getNode(0)
+
+    Print("Validating accounts before user accounts creation")
+    cluster.validateAccounts(None)
+
+    # create accounts via eosio as otherwise a bid is needed 
+    Print("Create new account %s via %s" % (testeraAccount.name, cluster.eosioAccount.name))
+    transId=node.createInitializeAccount(testeraAccount, cluster.eosioAccount, stakedDeposit=0, waitForTransBlock=False, exitOnError=True)
+
+    Print("Create new account %s via %s" % (currencyAccount.name, cluster.eosioAccount.name))
+    transId=node.createInitializeAccount(currencyAccount, cluster.eosioAccount, buyRAM=1000000, stakedDeposit=5000, exitOnError=True)
+
+    Print("Create new account %s via %s" % (exchangeAccount.name, cluster.eosioAccount.name))
+    transId=node.createInitializeAccount(exchangeAccount, cluster.eosioAccount, buyRAM=1000000, waitForTransBlock=True, exitOnError=True)
+
+    Print("Validating accounts after user accounts creation")
+    accounts=[testeraAccount, currencyAccount, exchangeAccount]
+    cluster.validateAccounts(accounts)
+
+    Print("Verify account %s" % (testeraAccount))
+    if not node.verifyAccount(testeraAccount):
+        errorExit("FAILURE - account creation failed.", raw=True)
+
+    transferAmount="97.5321 {0}".format(CORE_SYMBOL)
+    Print("Transfer funds %s from account %s to %s" % (transferAmount, defproduceraAccount.name, testeraAccount.name))
+    node.transferFunds(defproduceraAccount, testeraAccount, transferAmount, "test transfer")
+
+    expectedAmount=transferAmount
+    Print("Verify transfer, Expected: %s" % (expectedAmount))
+    actualAmount=node.getAccountEosBalanceStr(testeraAccount.name)
+    if expectedAmount != actualAmount:
+        cmdError("FAILURE - transfer failed")
+        errorExit("Transfer verification failed. Excepted %s, actual: %s" % (expectedAmount, actualAmount))
+
+    transferAmount="0.0100 {0}".format(CORE_SYMBOL)
+    Print("Force transfer funds %s from account %s to %s" % (
+        transferAmount, defproduceraAccount.name, testeraAccount.name))
+    node.transferFunds(defproduceraAccount, testeraAccount, transferAmount, "test transfer", force=True)
+
+    expectedAmount="97.5421 {0}".format(CORE_SYMBOL)
+    Print("Verify transfer, Expected: %s" % (expectedAmount))
+    actualAmount=node.getAccountEosBalanceStr(testeraAccount.name)
+    if expectedAmount != actualAmount:
+        cmdError("FAILURE - transfer failed")
+        errorExit("Transfer verification failed. Excepted %s, actual: %s" % (expectedAmount, actualAmount))
+
+    Print("Validating accounts after some user transactions")
+    accounts=[testeraAccount, currencyAccount, exchangeAccount]
+    cluster.validateAccounts(accounts)
+
+    transferAmount="97.5311 {0}".format(CORE_SYMBOL)
+    Print("Transfer funds %s from account %s to %s" % (
+        transferAmount, testeraAccount.name, currencyAccount.name))
+    trans=node.transferFunds(testeraAccount, currencyAccount, transferAmount, "test transfer a->b")
+    transId=Node.getTransId(trans)
+
+    expectedAmount="98.0311 {0}".format(CORE_SYMBOL) # 5000 initial deposit
+    Print("Verify transfer, Expected: %s" % (expectedAmount))
+    actualAmount=node.getAccountEosBalanceStr(currencyAccount.name)
+    if expectedAmount != actualAmount:
+        cmdError("FAILURE - transfer failed")
+        errorExit("Transfer verification failed. Excepted %s, actual: %s" % (expectedAmount, actualAmount))
+
+    Print("Validate last action for account %s" % (testeraAccount.name))
+    actions=node.getActions(testeraAccount, -1, -1, exitOnError=True)
+    try:
+        assert(actions["actions"][0]["action_trace"]["act"]["name"] == "transfer")
+    except (AssertionError, TypeError, KeyError) as _:
+        Print("Action validation failed. Actions: %s" % (actions))
+        raise
+
+    node.waitForTransInBlock(transId)
+
+    transaction=node.getTransaction(transId, exitOnError=True, delayedRetry=False)
+
+    typeVal=None
+    amountVal=None
+    key=""
+    try:
+        key="[traces][0][act][name]"
+        typeVal=  transaction["traces"][0]["act"]["name"]
+        key="[traces][0][act][data][quantity]"
+        amountVal=transaction["traces"][0]["act"]["data"]["quantity"]
+        amountVal=int(decimal.Decimal(amountVal.split()[0])*10000)
+    except (TypeError, KeyError) as e:
+        Print("transaction%s not found. Transaction: %s" % (key, transaction))
+        raise
+
+    if typeVal != "transfer" or amountVal != 975311:
+        errorExit("FAILURE - get transaction trans_id failed: %s %s %s" % (transId, typeVal, amountVal), raw=True)
+
+    Print("Bouncing nodes #00 and #01")
+    if cluster.bounce("00,01") is False:
+        cmdError("launcher bounce")
+        errorExit("Failed to bounce eos node.")
+
+    Print("Taking down node #02")
+    if cluster.down("02") is False:
+        cmdError("launcher down command")
+        errorExit("Failed to take down eos node.")
+
+    Print("Using bounce option to re-launch node #02")
+    if cluster.bounce("02") is False:
+        cmdError("launcher bounce")
+        errorExit("Failed to bounce eos node.")
+
+    p = re.compile('Assert')
+    errFileName="var/lib/node_00/stderr.txt"
+    assertionsFound=False
+    with open(errFileName) as errFile:
+        for line in errFile:
+            if p.search(line):
+                assertionsFound=True
+
+    if assertionsFound:
+        # Too many assertion logs, hard to validate how many are genuine. Make this a warning
+        #  for now, hopefully the logs will get cleaned up in future.
+        Print("WARNING: Asserts in var/lib/node_00/stderr.txt")
+        #errorExit("FAILURE - Assert in var/lib/node_00/stderr.txt")
+
+    Print("Validating accounts at end of test")
+    accounts=[testeraAccount, currencyAccount, exchangeAccount]
+    cluster.validateAccounts(accounts)
+
+    testSuccessful=True
+finally:
+    TestHelper.shutdown(cluster, walletMgr, testSuccessful, killEosInstances, killWallet, keepLogs, killAll, dumpErrorDetails)
+
+exit(0)

tests/nodeos_run_test.py

@@ -206,10 +206,10 @@ try:
     transId=node.createInitializeAccount(testeraAccount, cluster.eosioAccount, stakedDeposit=0, waitForTransBlock=False, exitOnError=True)
 
     Print("Create new account %s via %s" % (currencyAccount.name, cluster.eosioAccount.name))
-    transId=node.createInitializeAccount(currencyAccount, cluster.eosioAccount, stakedDeposit=5000, exitOnError=True)
+    transId=node.createInitializeAccount(currencyAccount, cluster.eosioAccount, buyRAM=1000000, stakedDeposit=5000, exitOnError=True)
 
     Print("Create new account %s via %s" % (exchangeAccount.name, cluster.eosioAccount.name))
-    transId=node.createInitializeAccount(exchangeAccount, cluster.eosioAccount, waitForTransBlock=True, exitOnError=True)
+    transId=node.createInitializeAccount(exchangeAccount, cluster.eosioAccount, buyRAM=1000000, waitForTransBlock=True, exitOnError=True)
 
     Print("Validating accounts after user accounts creation")
     accounts=[testeraAccount, currencyAccount, exchangeAccount]

tests/nodeos_under_min_avail_ram.py

@@ -81,7 +81,7 @@ try:
     maxRAMFlag="--chain-state-db-size-mb"
     maxRAMValue=1010
     extraNodeosArgs=" %s %d %s %d " % (minRAMFlag, minRAMValue, maxRAMFlag, maxRAMValue)
-    if cluster.launch(onlyBios=False, dontKill=dontKill, pnodes=totalNodes, totalNodes=totalNodes, totalProducers=totalNodes, extraNodeosArgs=extraNodeosArgs) is False:
+    if cluster.launch(onlyBios=False, dontKill=dontKill, pnodes=totalNodes, totalNodes=totalNodes, totalProducers=totalNodes, extraNodeosArgs=extraNodeosArgs, useBiosBootFile=False) is False:
         Utils.cmdError("launcher")
         errorExit("Failed to stand up eos cluster.")
 

tests/nodeos_voting_test.py

@@ -209,7 +209,7 @@ try:
     cluster.killall(allInstances=killAll)
     cluster.cleanup()
     Print("Stand up cluster")
-    if cluster.launch(prodCount=prodCount, onlyBios=False, dontKill=dontKill, pnodes=totalNodes, totalNodes=totalNodes, totalProducers=totalNodes*21, p2pPlugin=p2pPlugin) is False:
+    if cluster.launch(prodCount=prodCount, onlyBios=False, dontKill=dontKill, pnodes=totalNodes, totalNodes=totalNodes, totalProducers=totalNodes*21, p2pPlugin=p2pPlugin, useBiosBootFile=False) is False:
         Utils.cmdError("launcher")
         errorExit("Failed to stand up eos cluster.")