#!/usr/bin/env python3 from testUtils import Utils import testUtils import time from Cluster import Cluster from WalletMgr import WalletMgr from Node import BlockType from Node import Node from TestHelper import AppArgs from TestHelper import TestHelper import decimal import math import re import signal ############################################################### # nodeos_forked_chain_test # --dump-error-details # --keep-logs ############################################################### Print=Utils.Print from core_symbol import CORE_SYMBOL def analyzeBPs(bps0, bps1, expectDivergence): start=0 index=None length=len(bps0) firstDivergence=None errorInDivergence=False while start < length: bpsStr=None for i in range(start,length): bp0=bps0[i] bp1=bps1[i] if bpsStr is None: bpsStr="" else: bpsStr+=", " blockNum0=bp0["blockNum"] prod0=bp0["prod"] blockNum1=bp1["blockNum"] prod1=bp1["prod"] numDiff=True if blockNum0!=blockNum1 else False prodDiff=True if prod0!=prod1 else False if numDiff or prodDiff: index=i if firstDivergence is None: firstDivergence=min(blockNum0, blockNum1) if not expectDivergence: errorInDivergence=True break bpsStr+=str(blockNum0)+"->"+prod0 if index is None: return bpsStr0=None bpsStr2=None start=length for i in range(index,length): if bpsStr0 is None: bpsStr0="" bpsStr1="" else: bpsStr0+=", " bpsStr1+=", " bp0=bps0[i] bp1=bps1[i] blockNum0=bp0["blockNum"] prod0=bp0["prod"] blockNum1=bp1["blockNum"] prod1=bp1["prod"] numDiff="*" if blockNum0!=blockNum1 else "" prodDiff="*" if prod0!=prod1 else "" if not numDiff and not prodDiff: start=i index=None if expectDivergence: errorInDivergence=True break bpsStr0+=str(blockNum0)+numDiff+"->"+prod0+prodDiff bpsStr1+=str(blockNum1)+numDiff+"->"+prod1+prodDiff if errorInDivergence: msg="Failed analyzing block producers - " if expectDivergence: msg+="nodes indicate different block producers for the same blocks, but did not expect them to diverge." else: msg+="did not expect nodes to indicate different block producers for the same blocks." msg+="\n Matching Blocks= %s \n Diverging branch node0= %s \n Diverging branch node1= %s" % (bpsStr,bpsStr0,bpsStr1) Utils.errorExit(msg) return firstDivergence def getMinHeadAndLib(prodNodes): info0=prodNodes[0].getInfo(exitOnError=True) info1=prodNodes[1].getInfo(exitOnError=True) headBlockNum=min(int(info0["head_block_num"]),int(info1["head_block_num"])) libNum=min(int(info0["last_irreversible_block_num"]), int(info1["last_irreversible_block_num"])) return (headBlockNum, libNum) args = TestHelper.parse_args({"--prod-count","--dump-error-details","--keep-logs","-v","--leave-running","--clean-run", "--p2p-plugin","--wallet-port"}) Utils.Debug=args.v totalProducerNodes=2 totalNonProducerNodes=1 totalNodes=totalProducerNodes+totalNonProducerNodes maxActiveProducers=21 totalProducers=maxActiveProducers cluster=Cluster(walletd=True) dumpErrorDetails=args.dump_error_details keepLogs=args.keep_logs dontKill=args.leave_running prodCount=args.prod_count killAll=args.clean_run p2pPlugin=args.p2p_plugin walletPort=args.wallet_port walletMgr=WalletMgr(True, port=walletPort) testSuccessful=False killEosInstances=not dontKill killWallet=not dontKill WalletdName=Utils.EosWalletName ClientName="cleos" try: TestHelper.printSystemInfo("BEGIN") cluster.setWalletMgr(walletMgr) cluster.killall(allInstances=killAll) cluster.cleanup() Print("Stand up cluster") specificExtraNodeosArgs={} # producer nodes will be mapped to 0 through totalProducerNodes-1, so the number totalProducerNodes will be the non-producing node specificExtraNodeosArgs[totalProducerNodes]="--plugin eosio::test_control_api_plugin" # *** setup topogrophy *** # "bridge" shape connects defprocera through defproducerk (in node0) to each other and defproducerl through defproduceru (in node01) # and the only connection between those 2 groups is through the bridge node if cluster.launch(prodCount=prodCount, onlyBios=False, topo="bridge", pnodes=totalProducerNodes, totalNodes=totalNodes, totalProducers=totalProducers, p2pPlugin=p2pPlugin, useBiosBootFile=False, specificExtraNodeosArgs=specificExtraNodeosArgs) is False: Utils.cmdError("launcher") Utils.errorExit("Failed to stand up eos cluster.") Print("Validating system accounts after bootstrap") cluster.validateAccounts(None) # *** create accounts to vote in desired producers *** accounts=cluster.createAccountKeys(5) if accounts is None: Utils.errorExit("FAILURE - create keys") accounts[0].name="tester111111" accounts[1].name="tester222222" accounts[2].name="tester333333" accounts[3].name="tester444444" accounts[4].name="tester555555" testWalletName="test" Print("Creating wallet \"%s\"." % (testWalletName)) testWallet=walletMgr.create(testWalletName, [cluster.eosioAccount,accounts[0],accounts[1],accounts[2],accounts[3],accounts[4]]) for _, account in cluster.defProducerAccounts.items(): walletMgr.importKey(account, testWallet, ignoreDupKeyWarning=True) Print("Wallet \"%s\" password=%s." % (testWalletName, testWallet.password.encode("utf-8"))) # *** identify each node (producers and non-producing node) *** nonProdNode=None prodNodes=[] producers=[] for i in range(0, totalNodes): node=cluster.getNode(i) node.producers=Cluster.parseProducers(i) numProducers=len(node.producers) Print("node has producers=%s" % (node.producers)) if numProducers==0: if nonProdNode is None: nonProdNode=node nonProdNode.nodeNum=i else: Utils.errorExit("More than one non-producing nodes") else: for prod in node.producers: trans=node.regproducer(cluster.defProducerAccounts[prod], "http::/mysite.com", 0, waitForTransBlock=False, exitOnError=True) prodNodes.append(node) producers.extend(node.producers) # *** delegate bandwidth to accounts *** node=prodNodes[0] # create accounts via eosio as otherwise a bid is needed for account in accounts: Print("Create new account %s via %s" % (account.name, cluster.eosioAccount.name)) trans=node.createInitializeAccount(account, cluster.eosioAccount, stakedDeposit=0, waitForTransBlock=True, stakeNet=1000, stakeCPU=1000, buyRAM=1000, exitOnError=True) transferAmount="100000000.0000 {0}".format(CORE_SYMBOL) Print("Transfer funds %s from account %s to %s" % (transferAmount, cluster.eosioAccount.name, account.name)) node.transferFunds(cluster.eosioAccount, account, transferAmount, "test transfer", waitForTransBlock=True) trans=node.delegatebw(account, 20000000.0000, 20000000.0000, waitForTransBlock=True, exitOnError=True) # *** vote using accounts *** #verify nodes are in sync and advancing cluster.waitOnClusterSync(blockAdvancing=5) index=0 for account in accounts: Print("Vote for producers=%s" % (producers)) trans=prodNodes[index % len(prodNodes)].vote(account, producers, waitForTransBlock=True) index+=1 # *** Identify a block where production is stable *** #verify nodes are in sync and advancing cluster.waitOnClusterSync(blockAdvancing=5) blockNum=node.getNextCleanProductionCycle(trans) blockProducer=node.getBlockProducerByNum(blockNum) Print("Validating blockNum=%s, producer=%s" % (blockNum, blockProducer)) cluster.biosNode.kill(signal.SIGTERM) #advance to the next block of 12 lastBlockProducer=blockProducer while blockProducer==lastBlockProducer: blockNum+=1 blockProducer=node.getBlockProducerByNum(blockNum) # *** Identify what the production cycel is *** productionCycle=[] producerToSlot={} slot=-1 inRowCountPerProducer=12 while True: if blockProducer not in producers: Utils.errorExit("Producer %s was not one of the voted on producers" % blockProducer) productionCycle.append(blockProducer) slot+=1 if blockProducer in producerToSlot: Utils.errorExit("Producer %s was first seen in slot %d, but is repeated in slot %d" % (blockProducer, producerToSlot[blockProducer], slot)) producerToSlot[blockProducer]={"slot":slot, "count":0} lastBlockProducer=blockProducer while blockProducer==lastBlockProducer: producerToSlot[blockProducer]["count"]+=1 blockNum+=1 blockProducer=node.getBlockProducerByNum(blockNum) if producerToSlot[lastBlockProducer]["count"]!=inRowCountPerProducer: Utils.errorExit("Producer %s, in slot %d, expected to produce %d blocks but produced %d blocks" % (blockProducer, inRowCountPerProducer, producerToSlot[lastBlockProducer]["count"])) if blockProducer==productionCycle[0]: break output=None for blockProducer in productionCycle: if output is None: output="" else: output+=", " output+=blockProducer+":"+str(producerToSlot[blockProducer]["count"]) Print("ProductionCycle ->> {\n%s\n}" % output) #retrieve the info for all the nodes to report the status for each for node in cluster.getNodes(): node.getInfo() cluster.reportStatus() # *** Killing the "bridge" node *** Print("Sending command to kill \"bridge\" node to separate the 2 producer groups.") # block number to start expecting node killed after preKillBlockNum=nonProdNode.getBlockNum() preKillBlockProducer=nonProdNode.getBlockProducerByNum(preKillBlockNum) # kill at last block before defproducerl, since the block it is killed on will get propagated killAtProducer="defproducerk" nonProdNode.killNodeOnProducer(producer=killAtProducer, whereInSequence=(inRowCountPerProducer-1)) # *** Identify a highest block number to check while we are trying to identify where the divergence will occur *** # will search full cycle after the current block, since we don't know how many blocks were produced since retrieving # block number and issuing kill command postKillBlockNum=prodNodes[1].getBlockNum() blockProducers0=[] blockProducers1=[] libs0=[] libs1=[] lastBlockNum=max([preKillBlockNum,postKillBlockNum])+maxActiveProducers*inRowCountPerProducer actualLastBlockNum=None prodChanged=False nextProdChange=False #identify the earliest LIB to start identify the earliest block to check if divergent branches eventually reach concensus (headBlockNum, libNumAroundDivergence)=getMinHeadAndLib(prodNodes) for blockNum in range(preKillBlockNum,lastBlockNum): #avoiding getting LIB until my current block passes the head from the last time I checked if blockNum>headBlockNum: (headBlockNum, libNumAroundDivergence)=getMinHeadAndLib(prodNodes) # track the block number and producer from each producing node blockProducer0=prodNodes[0].getBlockProducerByNum(blockNum) blockProducer1=prodNodes[1].getBlockProducerByNum(blockNum) blockProducers0.append({"blockNum":blockNum, "prod":blockProducer0}) blockProducers1.append({"blockNum":blockNum, "prod":blockProducer1}) #in the case that the preKillBlockNum was also produced by killAtProducer, ensure that we have #at least one producer transition before checking for killAtProducer if not prodChanged: if preKillBlockProducer!=blockProducer0: prodChanged=True #since it is killing for the last block of killAtProducer, we look for the next producer change if not nextProdChange and prodChanged and blockProducer1==killAtProducer: nextProdChange=True elif nextProdChange and blockProducer1!=killAtProducer: actualLastBlockNum=blockNum break #if we diverge before identifying the actualLastBlockNum, then there is an ERROR if blockProducer0!=blockProducer1: Utils.errorExit("Groups reported different block producers for block number %d. %s != %s." % (blockNum,blockProducer0,blockProducer1)) # *** Analyze the producers leading up to the block after killing the non-producing node *** firstDivergence=analyzeBPs(blockProducers0, blockProducers1, expectDivergence=True) # Nodes should not have diverged till the last block if firstDivergence!=blockNum: Utils.errorExit("Expected to diverge at %s, but diverged at %s." % (firstDivergence, blockNum)) blockProducers0=[] blockProducers1=[] #verify that the non producing node is not alive (and populate the producer nodes with current getInfo data to report if #an error occurs) if nonProdNode.verifyAlive(): Utils.errorExit("Expected the non-producing node to have shutdown.") for prodNode in prodNodes: prodNode.getInfo() # *** Track the blocks from the divergence till there are 10*12 blocks on one chain and 10*12+1 on the other *** killBlockNum=blockNum lastBlockNum=killBlockNum+(maxActiveProducers - 1)*inRowCountPerProducer+1 # allow 1st testnet group to produce just 1 more block than the 2nd for blockNum in range(killBlockNum,lastBlockNum): blockProducer0=prodNodes[0].getBlockProducerByNum(blockNum) blockProducer1=prodNodes[1].getBlockProducerByNum(blockNum) blockProducers0.append({"blockNum":blockNum, "prod":blockProducer0}) blockProducers1.append({"blockNum":blockNum, "prod":blockProducer1}) # *** Analyze the producers from the divergence to the lastBlockNum and verify they stay diverged *** firstDivergence=analyzeBPs(blockProducers0, blockProducers1, expectDivergence=True) if firstDivergence!=killBlockNum: Utils.errorExit("Expected to diverge at %s, but diverged at %s." % (firstDivergence, killBlockNum)) blockProducers0=[] blockProducers1=[] # *** Relaunch the non-producing bridge node to connect the producing nodes again *** if not nonProdNode.relaunch(nonProdNode.nodeNum, None): errorExit("Failure - (non-production) node %d should have restarted" % (nonProdNode.nodeNum)) # *** Identify the producers from the saved LIB to the current highest head *** #ensure that the nodes have enough time to get in concensus, so wait for 3 producers to produce their complete round time.sleep(inRowCountPerProducer * 3 / 2) # ensure all blocks from the lib before divergence till the current head are now in consensus endBlockNum=max(prodNodes[0].getBlockNum(), prodNodes[1].getBlockNum()) for blockNum in range(libNumAroundDivergence,endBlockNum): blockProducer0=prodNodes[0].getBlockProducerByNum(blockNum) blockProducer1=prodNodes[1].getBlockProducerByNum(blockNum) blockProducers0.append({"blockNum":blockNum, "prod":blockProducer0}) blockProducers1.append({"blockNum":blockNum, "prod":blockProducer1}) # *** Analyze the producers from the saved LIB to the current highest head and verify they match now *** analyzeBPs(blockProducers0, blockProducers1, expectDivergence=False) blockProducers0=[] blockProducers1=[] testSuccessful=True finally: TestHelper.shutdown(cluster, walletMgr, testSuccessful, killEosInstances, killWallet, keepLogs, killAll, dumpErrorDetails) exit(0)