General tests cleanup. Enabling more segments of nodeos test. Tighten...

General tests cleanup. Enabling more segments of nodeos test. Tighten transaction finalization to narrow down the exact block containing the transaction. Tighter error handling. Add currency balance test. Improve transaction distribution validation mechanism.

tests/CMakeLists.txt

@@ -53,8 +53,8 @@ add_test(NAME nodeos_run_remote_test COMMAND tests/nodeos_run_remote_test.py -v
 # TODO: add_test(NAME p2p_sync_test_p2_d10 COMMAND tests/p2p_tests/sync/test.sh -p 2 -d 10 WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 # TODO: add_test(NAME message_storm COMMAND tests/p2p_tests/sync/test.sh -m -p 21 -n 21 -d 5 -l WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 # TODO: add_test(NAME trans_sync_across_mixed_cluster_test COMMAND tests/trans_sync_across_mixed_cluster_test.sh -p 1 -n 2 WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
-# TODO: add_test(NAME distributed-transactions-test COMMAND tests/distributed-transactions-test.py -p 1 -n 4 -v --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
-# TODO removed on slim: add_test(NAME distributed-transactions-remote-test COMMAND tests/distributed-transactions-remote-test.py -v --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME distributed-transactions-test COMMAND tests/distributed-transactions-test.py -p 1 -n 4 -v --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME distributed-transactions-remote-test COMMAND tests/distributed-transactions-remote-test.py -v --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 # TODO removed on slim: add_test(NAME restart-scenarios-test_resync COMMAND tests/restart-scenarios-test.py -c resync -p4 -v --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 # add_test(NAME restart-scenarios-test_replay COMMAND tests/restart-scenarios-test.py -c replay -p4 -v --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 # TODO: add_test(NAME consensus-validation-malicious-producers COMMAND tests/consensus-validation-malicious-producers.py -w 80 --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})

tests/distributed-transactions-remote-test.py

@@ -13,7 +13,7 @@ def errorExit(msg="", errorCode=1):
     Print("ERROR:", msg)
     exit(errorCode)
 
-pnodes=3
+pnodes=1
 # nodesFile="tests/sample-cluster-map.json"
 parser = argparse.ArgumentParser()
 parser.add_argument("-p", type=int, help="producing nodes count", default=pnodes)
@@ -36,7 +36,7 @@ killEosInstances=not dontKill
 topo="mesh"
 delay=1
 prodCount=1 # producers per producer node
-total_nodes=pnodes
+total_nodes=pnodes+3
 actualTest="tests/distributed-transactions-test.py"
 testSuccessful=False
 

tests/distributed-transactions-test.py

@@ -72,15 +72,14 @@ try:
         if cluster.launch(pnodes, total_nodes, topo=topo, delay=delay) is False:
             errorExit("Failed to stand up eos cluster.")
 
-        #exit(0)
         Print ("Wait for Cluster stabilization")
         # wait for cluster to start producing blocks
         if not cluster.waitOnClusterBlockNumSync(3):
             errorExit("Cluster never stabilized")
 
-    #exit(0)
-
     Print("Stand up EOS wallet keosd")
+    walletMgr.killall()
+    walletMgr.cleanup()
     if walletMgr.launch() is False:
         errorExit("Failed to stand up keosd.")
 
@@ -99,18 +98,15 @@ try:
     defproducerbAccount=cluster.defproducerbAccount
     eosioAccount=cluster.eosioAccount
 
-    # TBD: get account is currently failing. Enable when ready
-    # Print("Create accounts.")
-    # if not cluster.createAccounts(eosioAccount):
-    #     errorExit("Accounts creation failed.")
+    Print("Create accounts.")
+    if not cluster.createAccounts(eosioAccount):
+        errorExit("Accounts creation failed.")
 
-    # TBD: Known issue (Issue 2043) that 'get currency balance' doesn't return balance.
-    #  Uncomment when functional
-    # Print("Spread funds and validate")
-    # if not cluster.spreadFundsAndValidate(10):
-    #     errorExit("Failed to spread and validate funds.")
+    Print("Spread funds and validate")
+    if not cluster.spreadFundsAndValidate(10):
+        errorExit("Failed to spread and validate funds.")
 
-    # print("Funds spread validated")
+    print("Funds spread validated")
     
     testSuccessful=True
 finally:
@@ -126,6 +122,5 @@ finally:
         Print("Shut down the wallet and cleanup.")
         walletMgr.killall()
         walletMgr.cleanup()
-    pass
 
 exit(0)

tests/nodeos_run_test.py

@@ -325,10 +325,6 @@ try:
         Print("Last action validation failed. Actions: %s" % (actions))
         raise
 
-    # Pre-mature exit on slim branch. This will pushed futher out as code stablizes.
-    # testSuccessful=True
-    # exit(0)
-
     # This API (get accounts) is no longer supported (Issue 2876)
     # expectedAccounts=[testeraAccount.name, currencyAccount.name, exchangeAccount.name]
     # Print("Get accounts by key %s, Expected: %s" % (PUB_KEY3, expectedAccounts))
@@ -351,8 +347,8 @@ try:
     # if len(noMatch) > 0:
     #     errorExit("FAILURE - Accounts lookup by key %s. Expected: %s, Actual: %s" % (
     #         PUB_KEY1, expectedAccounts, actualAccounts), raw=True)
-    #
-    # This API (get servants) is no longer supported.
+
+    # This API (get servants) is no longer supported. (Issue 3160)
     # expectedServants=[testeraAccount.name, currencyAccount.name]
     # Print("Get %s servants, Expected: %s" % (defproduceraAccount.name, expectedServants))
     # actualServants=node.getServantsArr(defproduceraAccount.name)
@@ -403,15 +399,6 @@ try:
     if typeVal != "transfer" or amountVal != 975311:
         errorExit("FAILURE - get transaction trans_id failed: %s %s %s" % (transId, typeVal, amountVal), raw=True)
 
-    # This API (get transactions) is no longer supported.
-    # Print("Get transactions for account %s" % (testeraAccount.name))
-    # actualTransactions=node.getTransactionsArrByAccount(testeraAccount.name)
-    # if actualTransactions is None:
-    #     cmdError("%s get transactions testera11111" % (ClientName))
-    #     errorExit("Failed to get transactions by account %s" % (testeraAccount.name))
-    # if transId not in actualTransactions:
-    #     errorExit("FAILURE - get transactions testera11111 failed", raw=True)
-
     Print("Currency Contract Tests")
     Print("verify no contract in place")
     Print("Get code hash for account %s" % (currencyAccount.name))
@@ -456,47 +443,52 @@ try:
     data="{\"issuer\":\"currency1111\",\"maximum_supply\":\"100000.0000 CUR\",\"can_freeze\":\"0\",\"can_recall\":\"0\",\"can_whitelist\":\"0\"}"
     opts="--permission currency1111@active"
     trans=node.pushMessage(contract, action, data, opts)
-    if trans is None or not trans[0]:
-        errorExit("FAILURE - create action to currency1111 contract failed", raw=True)
+    try:
+        assert(trans)
+        assert(trans[0])
+    except (AssertionError, KeyError) as _:
+        Print("ERROR: Failed push create action to currency1111 contract assertion. %s" % (trans))
+        raise
 
     Print("push issue action to currency1111 contract")
     action="issue"
     data="{\"to\":\"currency1111\",\"quantity\":\"100000.0000 CUR\",\"memo\":\"issue\"}"
     opts="--permission currency1111@active"
     trans=node.pushMessage(contract, action, data, opts)
-    if trans is None or not trans[0]:
-        errorExit("FAILURE - issue action to currency1111 contract failed", raw=True)
+    try:
+        assert(trans)
+        assert(trans[0])
+    except (AssertionError, KeyError) as _:
+        Print("ERROR: Failed push issue action to currency1111 contract assertion. %s" % (trans))
+        raise
 
     Print("Verify currency1111 contract has proper initial balance (via get table)")
     contract="currency1111"
     table="accounts"
     row0=node.getTableRow(contract, currencyAccount.name, table, 0)
-    if row0 is None:
-        cmdError("%s get currency1111 table currency1111 account" % (ClientName))
-        errorExit("Failed to retrieve contract %s table %s" % (contract, table))
-
-    balanceKey="balance"
-    keyKey="key"
-    if row0[balanceKey] != "100000.0000 CUR":
-        errorExit("FAILURE - Wrong currency1111 balance", raw=True)
+    try:
+        assert(row0)
+        assert(row0["balance"] == "100000.0000 CUR")
+    except (AssertionError, KeyError) as _:
+        Print("ERROR: Failed get table row assertion. %s" % (row0))
+        raise
 
     Print("Verify currency1111 contract has proper initial balance (via get currency1111 balance)")
-    amountStr=node.getNodeAccountBalance("currency1111", currencyAccount.name)
+    amountStr=node.getTableAccountBalance("currency1111", currencyAccount.name)
 
     expected="100000.0000 CUR"
     actual=amountStr
     if actual != expected:
         errorExit("FAILURE - currency1111 balance check failed. Expected: %s, Recieved %s" % (expected, actual), raw=True)
 
-    # TBD: "get currency1111 stats is still not working. Enable when ready.
-    # Print("Verify currency1111 contract has proper total supply of CUR (via get currency1111 stats)")
-    # res=node.getCurrencyStats(contract, "CUR")
-    # if res is None or not ("supply" in res):
-    #     cmdError("%s get currency1111 stats" % (ClientName))
-    #     errorExit("Failed to retrieve CUR stats from contract %s" % (contract))
-    
-    # if res["supply"] != "100000.0000 CUR":
-    #     errorExit("FAILURE - get currency1111 stats failed", raw=True)
+    Print("Verify currency1111 contract has proper total supply of CUR (via get currency1111 stats)")
+    res=node.getCurrencyStats(contract, "CUR")
+    try:
+        assert(res)
+        assert(res["CUR"]["supply"] == "100000.0000 CUR")
+    except (AssertionError, KeyError) as _:
+        Print("ERROR: Failed get currecy stats assertion. %s" % (res))
+        raise
 
     Print("push transfer action to currency1111 contract")
     contract="currency1111"
@@ -516,20 +508,30 @@ try:
         errorExit("Failed to verify push message transaction id.")
 
     Print("read current contract balance")
-    amountStr=node.getNodeAccountBalance("currency1111", defproduceraAccount.name) 
+    amountStr=node.getTableAccountBalance("currency1111", defproduceraAccount.name)
 
     expected="0.0050 CUR"
     actual=amountStr
     if actual != expected:
         errorExit("FAILURE - Wrong currency1111 balance (expected=%s, actual=%s)" % (str(expected), str(actual)), raw=True)
 
-    amountStr=node.getNodeAccountBalance("currency1111", currencyAccount.name) 
+    amountStr=node.getTableAccountBalance("currency1111", currencyAccount.name)
 
     expected="99999.9950 CUR"
     actual=amountStr
     if actual != expected:
         errorExit("FAILURE - Wrong currency1111 balance (expected=%s, actual=%s)" % (str(expected), str(actual)), raw=True)
 
+    amountStr=node.getCurrencyBalance("currency1111", currencyAccount.name, "CUR")
+    try:
+        assert(actual)
+        assert(isinstance(actual, str))
+        actual=amountStr.strip()
+        assert(expected == actual)
+    except (AssertionError, KeyError) as _:
+        Print("ERROR: Failed get currecy balance assertion. (expected=<%s>, actual=<%s>)" % (str(expected), str(actual)))
+        raise
+
     Print("Test for block decoded packed transaction (issue 2932)")
     blockId=node.getBlockIdByTransId(transId)
     assert(blockId)

tests/testUtils.py

@@ -122,7 +122,6 @@ class Account(object):
 
     def __init__(self, name):
         self.name=name
-        self.balance=0
 
         self.ownerPrivateKey=None
         self.ownerPublicKey=None
@@ -469,29 +468,28 @@ class Node(object):
         blockId=self.getBlockIdByTransId(transId)
         return True if blockId else None
 
-    def getTransByBlockId(self, blockId, retry=True, silentErrors=False):
-        for _ in range(2):
-            cmd="%s %s" % (Utils.MongoPath, self.mongoEndpointArgs)
-            subcommand='db.Transactions.find( { "block_id": "%s" } )' % (blockId)
-            if Utils.Debug: Utils.Print("cmd: echo '%s' | %s" % (subcommand, cmd))
-            try:
-                trans=Node.runMongoCmdReturnJson(cmd.split(), subcommand, True)
-                if trans is not None:
-                    return trans
-            except subprocess.CalledProcessError as ex:
-                if not silentErrors:
-                    msg=ex.output.decode("utf-8")
-                    Utils.Print("ERROR: Exception during db get trans by blockId. %s" % (msg))
-                return None
-            if not retry:
-                break
-            if self.mongoSyncTime is not None:
-                if Utils.Debug: Utils.Print("cmd: sleep %d seconds" % (self.mongoSyncTime))
-                time.sleep(self.mongoSyncTime)
-
-        return None
-
+    # Disabling MongodDB funbction
+    # def getTransByBlockId(self, blockId, retry=True, silentErrors=False):
+    #     for _ in range(2):
+    #         cmd="%s %s" % (Utils.MongoPath, self.mongoEndpointArgs)
+    #         subcommand='db.Transactions.find( { "block_id": "%s" } )' % (blockId)
+    #         if Utils.Debug: Utils.Print("cmd: echo '%s' | %s" % (subcommand, cmd))
+    #         try:
+    #             trans=Node.runMongoCmdReturnJson(cmd.split(), subcommand, True)
+    #             if trans is not None:
+    #                 return trans
+    #         except subprocess.CalledProcessError as ex:
+    #             if not silentErrors:
+    #                 msg=ex.output.decode("utf-8")
+    #                 Utils.Print("ERROR: Exception during db get trans by blockId. %s" % (msg))
+    #             return None
+    #         if not retry:
+    #             break
+    #         if self.mongoSyncTime is not None:
+    #             if Utils.Debug: Utils.Print("cmd: sleep %d seconds" % (self.mongoSyncTime))
+    #             time.sleep(self.mongoSyncTime)
 
+    #     return None
 
     def getActionFromDb(self, transId, retry=True, silentErrors=False):
         for _ in range(2):
@@ -556,7 +554,7 @@ class Node(object):
 
         if stakedDeposit > 0:
             self.waitForTransIdOnNode(transId) # seems like account creation needs to be finlized before transfer can happen
-            trans = self.transferFunds(creatorAccount, account, "%0.04f %s" % (stakedDeposit/10000, CORE_SYMBOL), "init")
+            trans = self.transferFunds(creatorAccount, account, Node.currencyIntToStr(stakedDeposit, CORE_SYMBOL), "init")
             transId=Node.getTransId(trans)
 
         if waitForTransBlock and not self.waitForTransIdOnNode(transId):
@@ -626,8 +624,7 @@ class Node(object):
             Utils.Print("ERROR: Exception during table retrieval. %s" % (msg))
             return None
 
-    #def getNodeAccountEosBalance(self, scope):
-    def getNodeAccountBalance(self, contract, scope):
+    def getTableAccountBalance(self, contract, scope):
         assert(isinstance(contract, str))
         assert(isinstance(scope, str))
         table="accounts"
@@ -639,15 +636,38 @@ class Node(object):
             print("Transaction parsing failed. Transaction: %s" % (trans))
             raise
 
-    def getCurrencyStats(self, contract, symbol=""):
-        cmd="%s %s get currency0000 stats %s %s" % (Utils.EosClientPath, self.endpointArgs, contract, symbol)
+    def getCurrencyBalance(self, contract, account, symbol=CORE_SYMBOL):
+        """returns raw output from get currency balance e.g. '99999.9950 CUR'"""
+        assert(contract)
+        assert(isinstance(contract, str))
+        assert(account)
+        assert(isinstance(account, str))
+        assert(symbol)
+        assert(isinstance(symbol, str))
+        cmd="%s %s get currency balance %s %s %s" % (Utils.EosClientPath, self.endpointArgs, contract, account, symbol)
+        if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
+        try:
+            trans=Node.runCmdReturnStr(cmd)
+            return trans
+        except subprocess.CalledProcessError as ex:
+            msg=ex.output.decode("utf-8")
+            Utils.Print("ERROR: Exception during get currency stats. %s" % (msg))
+            return None
+
+    def getCurrencyStats(self, contract, symbol=CORE_SYMBOL):
+        """returns Json output from get currency stats."""
+        assert(contract)
+        assert(isinstance(contract, str))
+        assert(symbol)
+        assert(isinstance(symbol, str))
+        cmd="%s %s get currency stats %s %s" % (Utils.EosClientPath, self.endpointArgs, contract, symbol)
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
         try:
             trans=Node.runCmdReturnJson(cmd)
             return trans
         except subprocess.CalledProcessError as ex:
             msg=ex.output.decode("utf-8")
-            Utils.Print("ERROR: Exception during get currency0000 stats. %s" % (msg))
+            Utils.Print("ERROR: Exception during get currency stats. %s" % (msg))
             return None
 
     # Verifies account. Returns "get account" json return object
@@ -692,13 +712,17 @@ class Node(object):
         return ret
 
     # Trasfer funds. Returns "transfer" json return object
-    def transferFunds(self, source, destination, amount, memo="memo", force=False):
-        assert isinstance(amount, str)
+    def transferFunds(self, source, destination, amountStr, memo="memo", force=False, waitForTransBlock=False):
+        assert isinstance(amountStr, str)
+        assert(source)
+        assert(isinstance(source, Account))
+        assert(destination)
+        assert(isinstance(destination, Account))
 
         cmd="%s %s -v transfer -j %s %s" % (
             Utils.EosClientPath, self.endpointArgs, source.name, destination.name)
         cmdArr=cmd.split()
-        cmdArr.append(amount)
+        cmdArr.append(amountStr)
         cmdArr.append(memo)
         if force:
             cmdArr.append("-f")
@@ -707,34 +731,80 @@ class Node(object):
         trans=None
         try:
             trans=Node.__runCmdArrReturnJson(cmdArr)
-            return trans
         except subprocess.CalledProcessError as ex:
             msg=ex.output.decode("utf-8")
             Utils.Print("ERROR: Exception during funds transfer. %s" % (msg))
             return None
 
-    def validateSpreadFundsOnNode(self, adminAccount, accounts, expectedTotal):
-        actualTotal=self.getAccountEosBalance(adminAccount.name)
-        for account in accounts:
-            fund = self.getAccountEosBalance(account.name)
-            if fund != account.balance:
-                Utils.Print("ERROR: validateSpreadFunds> Expected: %d, actual: %d for account %s" %
-                        (account.balance, fund, account.name))
-                return False
-            actualTotal += fund
+        assert(trans)
+        transId=Node.getTransId(trans)
+        if waitForTransBlock and not self.waitForTransIdOnNode(transId):
+            return None
+
+        return trans
 
-        if actualTotal != expectedTotal:
-            Utils.Print("ERROR: validateSpreadFunds> Expected total: %d , actual: %d" % (
-                expectedTotal, actualTotal))
+    @staticmethod
+    def currencyStrToInt(balanceStr):
+        """Converts currency string of form "12.3456 SYS" to int 123456"""
+        assert(isinstance(balanceStr, str))
+        balanceStr=balanceStr.split()[0]
+        #balance=int(decimal.Decimal(balanceStr[1:])*10000)
+        balance=int(decimal.Decimal(balanceStr)*10000)
+
+        return balance
+
+    @staticmethod
+    def currencyIntToStr(balance, symbol):
+        """Converts currency int of form 123456 to string "12.3456 SYS" where SYS is symbol string"""
+        assert(isinstance(balance, int))
+        assert(isinstance(symbol, str))
+        balanceStr="%.04f %s" % (balance/10000.0, symbol)
+
+        return balanceStr
+
+    def validateFunds(self, initialBalances, transferAmount, source, accounts):
+        """Validate each account has the expected SYS balance. Validate cumulative balance matches expectedTotal."""
+        assert(source)
+        assert(isinstance(source, Account))
+        assert(accounts)
+        assert(isinstance(accounts, list))
+        assert(len(accounts) > 0)
+        assert(initialBalances)
+        assert(isinstance(initialBalances, dict))
+        assert(isinstance(transferAmount, int))
+
+        currentBalances=self.getEosBalances([source] + accounts)
+        assert(currentBalances)
+        assert(isinstance(currentBalances, dict))
+        assert(len(initialBalances) == len(currentBalances))
+
+        if len(currentBalances) != len(initialBalances):
+            Utils.Print("ERROR: validateFunds> accounts length mismatch. Initial: %d, current: %d" % (len(initialBalances), len(currentBalances)))
             return False
 
-        return True
+        for key, value in currentBalances.items():
+            initialBalance = initialBalances[key]
+            assert(initialBalances)
+            expectedInitialBalance = value - transferAmount
+            if key is source:
+                expectedInitialBalance = value + (transferAmount*len(accounts))
+
+            if (initialBalance != expectedInitialBalance):
+                Utils.Print("ERROR: validateFunds> Expected: %d, actual: %d for account %s" %
+                            (expectedInitialBalance, initialBalance, key.name))
+                return False
+
+    def getEosBalances(self, accounts):
+        """Returns a dictionary with account balances keyed by accounts"""
+        assert(accounts)
+        assert(isinstance(accounts, list))
 
-    def getSystemBalance(self, adminAccount, accounts):
-        balance=self.getAccountEosBalance(adminAccount.name)
+        balances={}
         for account in accounts:
-            balance += self.getAccountEosBalance(account.name)
-        return balance
+            balance = self.getAccountEosBalance(account.name)
+            balances[account]=balance
+
+        return balances
 
     # Gets accounts mapped to key. Returns json object
     def getAccountsByKey(self, key):
@@ -789,10 +859,10 @@ class Node(object):
         return servants
 
     def getAccountEosBalanceStr(self, scope):
-        """Returns EOS currency0000 account balance from cleos get table command. Returned balance is string following syntax "98.0311 SYS". """
+        """Returns SYS currency0000 account balance from cleos get table command. Returned balance is string following syntax "98.0311 SYS". """
         assert isinstance(scope, str)
         if not self.enableMongo:
-            amount=self.getNodeAccountBalance("eosio.token", scope)
+            amount=self.getTableAccountBalance("eosio.token", scope)
             if Utils.Debug: Utils.Print("getNodeAccountEosBalance %s %s" % (scope, amount))
             assert isinstance(amount, str)
             return amount
@@ -809,34 +879,11 @@ class Node(object):
         return None
 
     def getAccountEosBalance(self, scope):
-        """Returns EOS currency0000 account balance from cleos get table command. Returned balance is an integer e.g. 980311. """
+        """Returns SYS currency0000 account balance from cleos get table command. Returned balance is an integer e.g. 980311. """
         balanceStr=self.getAccountEosBalanceStr(scope)
-        balanceStr=balanceStr.split()[0]
-        balance=int(decimal.Decimal(balanceStr[1:])*10000)
+        balance=Node.currencyStrToInt(balanceStr)
         return balance
 
-    # transactions lookup by id. Returns json object
-    def getTransactionsByAccount(self, name):
-        cmd="%s %s get transactions -j %s" % (Utils.EosClientPath, self.endpointArgs, name)
-        if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
-        try:
-            trans=Node.runCmdReturnJson(cmd)
-            return trans
-        except subprocess.CalledProcessError as ex:
-            msg=ex.output.decode("utf-8")
-            Utils.Print("ERROR: Exception during transactions by account retrieval. %s" % (msg))
-            return None
-
-    # transactions lookup by id. Returns list of transaction ids
-    def getTransactionsArrByAccount(self, name):
-        trans=self.getTransactionsByAccount(name)
-        transactions=trans["transactions"]
-        transArr=[]
-        for transaction in transactions:
-            transId=transaction["transaction_id"]
-            transArr.append(transId)
-        return transArr
-
     def getAccountCodeHash(self, account):
         cmd="%s %s get code %s" % (Utils.EosClientPath, self.endpointArgs, account)
         if Utils.Debug: Utils.Print("cmd: %s" % (cmd))
@@ -1242,7 +1289,6 @@ class Cluster(object):
     __chainSyncStrategy=None
     __WalletName="MyWallet"
     __localHost="localhost"
-    __lastTrans=None
     __BiosHost="localhost"
     __BiosPort=8788
     
@@ -1570,29 +1616,33 @@ class Cluster(object):
 
     # Spread funds across accounts with transactions spread through cluster nodes.
     #  Validate transactions are synchronized on root node
-    def spreadFunds(self, amount=1):
-        Utils.Print("len(self.accounts): %d" % (len(self.accounts)))
-        if len(self.accounts) == 0:
-            return True
+    def spreadFunds(self, source, accounts, amount=1):
+        assert(source)
+        assert(isinstance(source, Account))
+        assert(accounts)
+        assert(isinstance(accounts, list))
+        assert(len(accounts) > 0)
+        Utils.Print("len(accounts): %d" % (len(accounts)))
 
-        count=len(self.accounts)
+        count=len(accounts)
         transferAmount=(count*amount)+amount
+        transferAmountStr=Node.currencyIntToStr(transferAmount, CORE_SYMBOL)
         node=self.nodes[0]
-        fromm=self.defproduceraAccount
-        to=self.accounts[0]
-        Utils.Print("Transfer %d units from account %s to %s on eos server port %d" % (
-            transferAmount, fromm.name, to.name, node.port))
-        trans=node.transferFunds(fromm, to, transferAmount)
+        fromm=source
+        to=accounts[0]
+        Utils.Print("Transfer %s units from account %s to %s on eos server port %d" % (
+            transferAmountStr, fromm.name, to.name, node.port))
+        trans=node.transferFunds(fromm, to, transferAmountStr)
+        assert(trans)
         transId=Node.getTransId(trans)
         if transId is None:
             return False
-        self.__lastTrans=transId
+
         if Utils.Debug: Utils.Print("Funds transfered on transaction id %s." % (transId))
-        self.accounts[0].balance += transferAmount
 
         nextEosIdx=-1
         for i in range(0, count):
-            account=self.accounts[i]
+            account=accounts[i]
             nextInstanceFound=False
             for _ in range(0, count):
                 #Utils.Print("nextEosIdx: %d, n: %d" % (nextEosIdx, n))
@@ -1614,21 +1664,18 @@ class Cluster(object):
                 return False
 
             transferAmount -= amount
+            transferAmountStr=Node.currencyIntToStr(transferAmount, CORE_SYMBOL)
             fromm=account
-            to=self.accounts[i+1] if i < (count-1) else self.defproduceraAccount
-            Utils.Print("Transfer %d units from account %s to %s on eos server port %d." %
-                    (transferAmount, fromm.name, to.name, node.port))
+            to=accounts[i+1] if i < (count-1) else source
+            Utils.Print("Transfer %s units from account %s to %s on eos server port %d." %
+                    (transferAmountStr, fromm.name, to.name, node.port))
 
-            trans=node.transferFunds(fromm, to, transferAmount)
+            trans=node.transferFunds(fromm, to, transferAmountStr)
             transId=Node.getTransId(trans)
             if transId is None:
                 return False
-            self.__lastTrans=transId
-            if Utils.Debug: Utils.Print("Funds transfered on block num %s." % (transId))
 
-            self.accounts[i].balance -= transferAmount
-            if i < (count-1):
-                self.accounts[i+1].balance += transferAmount
+            if Utils.Debug: Utils.Print("Funds transfered on block num %s." % (transId))
 
         # As an extra step wait for last transaction on the root node
         node=self.nodes[0]
@@ -1639,30 +1686,47 @@ class Cluster(object):
 
         return True
 
-    def validateSpreadFunds(self, expectedTotal):
+    def validateSpreadFunds(self, initialBalances, transferAmount, source, accounts):
+        """Given initial Balances, will validate each account has the expected balance based upon transferAmount.
+        This validation is repeated against every node in the cluster."""
+        assert(source)
+        assert(isinstance(source, Account))
+        assert(accounts)
+        assert(isinstance(accounts, list))
+        assert(len(accounts) > 0)
+        assert(initialBalances)
+        assert(isinstance(initialBalances, dict))
+        assert(isinstance(transferAmount, int))
+
         for node in self.nodes:
-            if not node.killed:
-                if Utils.Debug: Utils.Print("Validate funds on %s server port %d." %
-                                            (Utils.EosServerName, node.port))
-                if node.validateSpreadFundsOnNode(self.defproduceraAccount, self.accounts, expectedTotal) is False:
-                    Utils.Print("ERROR: Failed to validate funds on eos node port: %d" % (node.port))
-                    return False
+            if node.killed:
+                continue
+
+            if Utils.Debug: Utils.Print("Validate funds on %s server port %d." %
+                                        (Utils.EosServerName, node.port))
+
+            if node.validateFunds(initialBalances, transferAmount, source, accounts) is False:
+                Utils.Print("ERROR: Failed to validate funds on eos node port: %d" % (node.port))
+                return False
 
         return True
 
-    def spreadFundsAndValidate(self, amount=1):
-        if Utils.Debug: Utils.Print("Get system balance.")
-        initialFunds=self.nodes[0].getSystemBalance(self.defproduceraAccount, self.accounts)
-        if Utils.Debug: Utils.Print("Initial system balance: %d" % (initialFunds))
+    def spreadFundsAndValidate(self, transferAmount=1):
+        """Sprays 'transferAmount' funds across configured accounts and validates action. The spray is done in a trickle down fashion with account 1
+        receiving transferAmount*n SYS and forwarding x-transferAmount funds. Transfer actions are spread round-robin across the cluster to vaidate system cohesiveness."""
+
+        if Utils.Debug: Utils.Print("Get initial system balances.")
+        initialBalances=self.nodes[0].getEosBalances([self.defproduceraAccount] + self.accounts)
+        assert(initialBalances)
+        assert(isinstance(initialBalances, dict))
 
-        if False == self.spreadFunds(amount):
+        if False == self.spreadFunds(self.defproduceraAccount, self.accounts, transferAmount):
             Utils.Print("ERROR: Failed to spread funds across nodes.")
             return False
 
-        Utils.Print("Funds spread across all accounts")
+        Utils.Print("Funds spread across all accounts. Noew validate funds")
 
-        Utils.Print("Validate funds.")
-        if False == self.validateSpreadFunds(initialFunds):
+        if False == self.validateSpreadFunds(initialBalances, transferAmount, self.defproduceraAccount, self.accounts):
             Utils.Print("ERROR: Failed to validate funds transfer across nodes.")
             return False
 
@@ -1681,8 +1745,8 @@ class Cluster(object):
 
         node.validateAccounts(myAccounts)
 
-    # create account, verify account and return transaction id
     def createAccountAndVerify(self, account, creator, stakedDeposit=1000):
+        """create account, verify account and return transaction id"""
         assert(len(self.nodes) > 0)
         node=self.nodes[0]
         trans=node.createInitializeAccount(account, creator, stakedDeposit)
@@ -1776,7 +1840,7 @@ class Cluster(object):
 
     @staticmethod
     def bootstrap(totalNodes, prodCount, biosHost, biosPort, dontKill=False, onlyBios=False):
-        """Create 'prodCount' init accounts and deposits 10000000000 EOS in each. If prodCount is -1 will initialize all possible producers.
+        """Create 'prodCount' init accounts and deposits 10000000000 SYS in each. If prodCount is -1 will initialize all possible producers.
         Ensure nodes are inter-connected prior to this call. One way to validate this will be to check if every node has block 1."""
 
         Utils.Print("Starting cluster bootstrap.")