Added integration test to verify that nodeos will shutdown if minimum RAM...

Added integration test to verify that nodeos will shutdown if minimum RAM setting hit and can restart with higher RAM. GH #4812

tests/nodeos_under_min_avail_ram.py

+#!/usr/bin/env python3
+
+from core_symbol import CORE_SYMBOL
+from Cluster import Cluster
+from WalletMgr import WalletMgr
+from Node import Node
+from TestHelper import TestHelper
+from testUtils import Utils
+import testUtils
+import time
+
+import decimal
+import math
+import re
+
+class NamedAccounts:
+
+    def __init__(self, cluster, numAccounts):
+        Print("NamedAccounts %d" % (numAccounts))
+        self.numAccounts=numAccounts
+        self.accounts=cluster.createAccountKeys(numAccounts)
+        if self.accounts is None:
+            errorExit("FAILURE - create keys")
+        accountNum = 0
+        for account in self.accounts:
+            Print("NamedAccounts Name for %d" % (accountNum))
+            account.name=self.setName(accountNum)
+            accountNum+=1
+
+    def setName(self, num):
+        retStr="test"
+        digits=[]
+        maxDigitVal=5
+        maxDigits=8
+        temp=num
+        while len(digits) < maxDigits:
+            digit=(num % maxDigitVal)+1
+            num=int(num/maxDigitVal)
+            digits.append(digit)
+
+        digits.reverse()
+        for digit in digits:
+            retStr=retStr+str(digit)
+
+        Print("NamedAccounts Name for %d is %s" % (temp, retStr))
+        return retStr
+
+###############################################################
+# nodeos_voting_test
+# --dump-error-details <Upon error print etc/eosio/node_*/config.ini and var/lib/node_*/stderr.log to stdout>
+# --keep-logs <Don't delete var/lib/node_* folders upon test completion>
+###############################################################
+Print=Utils.Print
+errorExit=Utils.errorExit
+
+args = TestHelper.parse_args({"--dump-error-details","--keep-logs","-v","--leave-running","--clean-run"})
+Utils.Debug=args.v
+totalNodes=4
+cluster=Cluster(walletd=True)
+dumpErrorDetails=args.dump_error_details
+keepLogs=args.keep_logs
+dontKill=args.leave_running
+killAll=args.clean_run
+
+walletMgr=WalletMgr(True)
+testSuccessful=False
+killEosInstances=not dontKill
+killWallet=not dontKill
+
+WalletdName="keosd"
+ClientName="cleos"
+
+try:
+    TestHelper.printSystemInfo("BEGIN")
+
+    cluster.killall(allInstances=killAll)
+    cluster.cleanup()
+    Print("Stand up cluster")
+    minRAMFlag="--chain-state-db-guard-size-mb"
+    minRAMValue=1002
+    maxRAMFlag="--chain-state-db-size-mb"
+    maxRAMValue=1010
+    extraNodeosArgs=" %s %d %s %d " % (minRAMFlag, minRAMValue, maxRAMFlag, maxRAMValue)
+    if cluster.launch(onlyBios=False, dontKill=dontKill, pnodes=totalNodes, totalNodes=totalNodes, totalProducers=totalNodes*21, extraNodeosArgs=extraNodeosArgs) is False:
+        Utils.cmdError("launcher")
+        errorExit("Failed to stand up eos cluster.")
+
+    Print("Validating system accounts after bootstrap")
+    cluster.validateAccounts(None)
+
+    Print("creating accounts")
+    namedAccounts=NamedAccounts(cluster,10)
+    accounts=namedAccounts.accounts
+
+    testWalletName="test"
+
+    Print("Creating wallet \"%s\"." % (testWalletName))
+    walletMgr.killall(allInstances=killAll)
+    walletMgr.cleanup()
+    if walletMgr.launch() is False:
+        Utils.cmdError("%s" % (WalletdName))
+        errorExit("Failed to stand up eos walletd.")
+
+    testWallet=walletMgr.create(testWalletName, [cluster.eosioAccount])
+    if testWallet is None:
+        Utils.cmdError("eos wallet create")
+        errorExit("Failed to create wallet %s." % (testWalletName))
+
+    for _, account in cluster.defProducerAccounts.items():
+        walletMgr.importKey(account, testWallet, ignoreDupKeyWarning=True)
+
+    Print("Wallet \"%s\" password=%s." % (testWalletName, testWallet.password.encode("utf-8")))
+
+    nodes=[]
+    nodes.append(cluster.getNode(0))
+    if nodes[0] is None:
+        errorExit("Cluster in bad state, received None node")
+    nodes.append(cluster.getNode(1))
+    if nodes[1] is None:
+        errorExit("Cluster in bad state, received None node")
+    nodes.append(cluster.getNode(2))
+    if nodes[2] is None:
+        errorExit("Cluster in bad state, received None node")
+    nodes.append(cluster.getNode(3))
+    if nodes[3] is None:
+        errorExit("Cluster in bad state, received None node")
+
+
+    for account in accounts:
+        walletMgr.importKey(account, testWallet)
+
+    # create accounts via eosio as otherwise a bid is needed
+    for account in accounts:
+        Print("Create new account %s via %s" % (account.name, cluster.eosioAccount.name))
+        trans=nodes[0].createInitializeAccount(account, cluster.eosioAccount, stakedDeposit=500000, waitForTransBlock=False, stakeNet=50000, stakeCPU=50000, buyRAM=50000)
+        if trans is None:
+            Utils.cmdError("%s create account" % (account.name))
+            errorExit("Failed to create account %s" % (account.name))
+        transferAmount="70000000.0000 {0}".format(CORE_SYMBOL)
+        Print("Transfer funds %s from account %s to %s" % (transferAmount, cluster.eosioAccount.name, account.name))
+        if nodes[0].transferFunds(cluster.eosioAccount, account, transferAmount, "test transfer") is None:
+            errorExit("Failed to transfer funds %d from account %s to %s" % (
+                transferAmount, cluster.eosioAccount.name, account.name))
+        trans=nodes[0].delegatebw(account, 1000000.0000, 68000000.0000) 
+        if trans is None:
+            Utils.cmdError("delegate bandwidth for %s" % (account.name))
+            errorExit("Failed to delegate bandwidth for %s" % (account.name))
+
+    contractAccount=cluster.createAccountKeys(1)[0]
+    contractAccount.name="contracttest"
+    walletMgr.importKey(contractAccount, testWallet)
+    Print("Create new account %s via %s" % (contractAccount.name, cluster.eosioAccount.name))
+    trans=nodes[0].createInitializeAccount(contractAccount, cluster.eosioAccount, stakedDeposit=500000, waitForTransBlock=False, stakeNet=50000, stakeCPU=50000, buyRAM=50000)
+    if trans is None:
+        Utils.cmdError("%s create account" % (contractAccount.name))
+        errorExit("Failed to create account %s" % (contractAccount.name))
+    transferAmount="90000000.0000 {0}".format(CORE_SYMBOL)
+    Print("Transfer funds %s from account %s to %s" % (transferAmount, cluster.eosioAccount.name, contractAccount.name))
+    if nodes[0].transferFunds(cluster.eosioAccount, contractAccount, transferAmount, "test transfer") is None:
+        errorExit("Failed to transfer funds %d from account %s to %s" % (
+            transferAmount, cluster.eosioAccount.name, contractAccount.name))
+    trans=nodes[0].delegatebw(contractAccount, 1000000.0000, 88000000.0000) 
+    if trans is None:
+        Utils.cmdError("delegate bandwidth for %s" % (contractAccount.name))
+        errorExit("Failed to delegate bandwidth for %s" % (contractAccount.name))
+
+    contractDir="contracts/integration_test"
+    wastFile="contracts/integration_test/integration_test.wast"
+    abiFile="contracts/integration_test/integration_test.abi"
+    Print("Publish contract")
+    trans=nodes[0].publishContract(contractAccount.name, contractDir, wastFile, abiFile, waitForTransBlock=True)
+    if trans is None:
+        cmdError("%s set contract %s" % (ClientName, contractAccount.name))
+        errorExit("Failed to publish contract.")
+
+    contract=contractAccount.name
+    Print("push create action to %s contract" % (contract))
+    action="store"
+    numAmount=5000
+    keepProcessing=True
+    count=0
+    while keepProcessing:
+        numAmount+=1
+        for fromIndex in range(namedAccounts.numAccounts):
+            count+=1
+            toIndex=fromIndex+1
+            if toIndex==namedAccounts.numAccounts:
+                toIndex=0
+            fromAccount=accounts[fromIndex]
+            toAccount=accounts[toIndex]
+            data="{\"from\":\"%s\",\"to\":\"%s\",\"num\":%d}" % (fromAccount.name, toAccount.name, numAmount)
+            opts="--permission %s@active --permission %s@active --expiration 90" % (contract, fromAccount.name)
+            try:
+                trans=nodes[0].pushMessage(contract, action, data, opts)
+                if trans is None or not trans[0]:
+                    Print("Failed to push create action to eosio contract. sleep for 60 seconds")
+                    time.sleep(60)
+                time.sleep(1)
+            except TypeError as ex:
+                keepProcessing=False
+                break
+
+    #spread the actions to all accounts, to use each accounts tps bandwidth
+    fromIndexStart=fromIndex+1 if fromIndex+1<namedAccounts.numAccounts else 0
+
+    if count < 5 or count > 15:
+        strMsg="little" if count < 15 else "much"
+        Utils.cmdError("Was able to send %d store actions which was too %s" % (count, strMsg))
+        errorExit("Incorrect number of store actions sent")
+
+    # Make sure all the nodes are shutdown (may take a little while for this to happen, so making multiple passes)
+    allDone=False
+    count=0
+    while not allDone:
+        allDone=True
+        for node in nodes:
+            if node.verifyAlive():
+                allDone=False
+        if not allDone:
+            time.sleep(5)
+        if ++count>5:
+            Utils.cmdError("All Nodes should have died")
+            errorExit("Failure - All Nodes should have died")
+
+    Print("relaunch nodes with new capacity")
+    addOrSwapFlags={}
+    numNodes=len(nodes)
+    maxRAMValue+=2
+    currentMinimumMaxRAM=maxRAMValue
+    for i in range(numNodes):
+        addOrSwapFlags[maxRAMFlag]=str(maxRAMValue)
+        if i==numNodes-1:
+            addOrSwapFlags["--enable-stale-production"]=""   # just enable stale production for the first node
+        nodeIndex=numNodes-i-1
+        if not nodes[nodeIndex].relaunch(nodeIndex, "", newChain=False, addOrSwapFlags=addOrSwapFlags):
+            Utils.cmdError("Failed to restart node0 with new capacity %s" % (maxRAMValue))
+            errorExit("Failure - Node should have restarted")
+        addOrSwapFlags={}
+        maxRAMValue=currentMinimumMaxRAM+30
+
+    time.sleep(10)
+    for i in range(numNodes):
+        if not nodes[i].verifyAlive():
+            Utils.cmdError("Node %d should be alive" % (i))
+            errorExit("Failure - All Nodes should be alive")
+
+    Print("push more actions to %s contract" % (contract))
+    action="store"
+    keepProcessing=True
+    count=0
+    while keepProcessing and count < 40:
+        Print("Send %s" % (action))
+        numAmount+=1
+        for fromIndexOffset in range(namedAccounts.numAccounts):
+            count+=1
+            fromIndex=fromIndexStart+fromIndexOffset
+            if fromIndex>=namedAccounts.numAccounts:
+                fromIndex-=namedAccounts.numAccounts 
+            toIndex=fromIndex+1
+            if toIndex==namedAccounts.numAccounts:
+                toIndex=0
+            fromAccount=accounts[fromIndex]
+            toAccount=accounts[toIndex]
+            data="{\"from\":\"%s\",\"to\":\"%s\",\"num\":%d}" % (fromAccount.name, toAccount.name, numAmount)
+            opts="--permission %s@active --permission %s@active --expiration 90" % (contract, fromAccount.name)
+            try:
+                trans=nodes[0].pushMessage(contract, action, data, opts)
+                if trans is None or not trans[0]:
+                    Print("Failed to push create action to eosio contract. sleep for 60 seconds")
+                    time.sleep(60)
+                time.sleep(1)
+            except TypeError as ex:
+                Print("Failed to send %s" % (action))
+
+            if not nodes[len(nodes)-1].verifyAlive():
+                keepProcessing=False
+                break
+
+    if keepProcessing:
+        Utils.cmdError("node[%d] never shutdown" % (numNodes-1))
+        errorExit("Failure - Node should be shutdown")
+
+    for i in range(numNodes):
+        # only the last node should be dead
+        if not nodes[i].verifyAlive() and i<numNodes-1:
+            Utils.cmdError("Node %d should be alive" % (i))
+            errorExit("Failure - Node should be alive")
+
+    Print("relaunch node with even more capacity")
+    addOrSwapFlags={}
+
+    time.sleep(10)
+    maxRAMValue=currentMinimumMaxRAM+5
+    currentMinimumMaxRAM=maxRAMValue
+    addOrSwapFlags[maxRAMFlag]=str(maxRAMValue)
+    if not nodes[len(nodes)-1].relaunch(nodeIndex, "", newChain=False, addOrSwapFlags=addOrSwapFlags):
+        Utils.cmdError("Failed to restart node i with new capacity %s" % (numNodes-1, maxRAMValue))
+        errorExit("Failure - Node should have restarted")
+    addOrSwapFlags={}
+
+    time.sleep(10)
+    allDone=True
+    for node in nodes:
+        if not node.verifyAlive():
+            Utils.cmdError("All Nodes should be alive")
+            errorExit("Failure - All Nodes should be alive")
+
+    Print("Send 1 more action to every node")
+    numAmount+=1
+    for fromIndexOffset in range(namedAccounts.numAccounts):
+        # just sending one node to each
+        if fromIndexOffset>=len(nodes):
+           break
+        fromIndex=fromIndexStart+fromIndexOffset
+        if fromIndex>=namedAccounts.numAccounts:
+            fromIndex-=namedAccounts.numAccounts 
+        toIndex=fromIndex+1
+        if toIndex==namedAccounts.numAccounts:
+            toIndex=0
+        fromAccount=accounts[fromIndex]
+        toAccount=accounts[toIndex]
+        node=nodes[fromIndexOffset]
+        data="{\"from\":\"%s\",\"to\":\"%s\",\"num\":%d}" % (fromAccount.name, toAccount.name, numAmount)
+        opts="--permission %s@active --permission %s@active --expiration 90" % (contract, fromAccount.name)
+        try:
+            trans=nodes[0].pushMessage(contract, action, data, opts)
+            if trans is None or not trans[0]:
+                Print("Failed to push create action to eosio contract. sleep for 60 seconds")
+                time.sleep(60)
+                continue
+            time.sleep(1)
+        except TypeError as ex:
+            Utils.cmdError("Failed to send %s action to node %d" % (fromAccount, fromIndexOffset, action))
+            errorExit("Failure - send %s action should have succeeded" % (action))
+
+    time.sleep(10)
+    Print("Check nodes are alive")
+    allDone=True
+    for node in nodes:
+        if not node.verifyAlive():
+            Utils.cmdError("All Nodes should be alive")
+            errorExit("Failure - All Nodes should be alive")
+
+    testSuccessful=True
+finally:
+    TestHelper.shutdown(cluster, walletMgr, testSuccessful, killEosInstances, killWallet, keepLogs, killAll, dumpErrorDetails)
+
+exit(0)