Add new test for dirty db flag stickiness.

tests/CMakeLists.txt

@@ -34,6 +34,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/testUtils.py ${CMAKE_CURRENT_BINARY_D
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_run_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_run_test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_run_remote_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_run_remote_test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/consensus-validation-malicious-producers.py ${CMAKE_CURRENT_BINARY_DIR}/consensus-validation-malicious-producers.py COPYONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/validate-dirty-db.py ${CMAKE_CURRENT_BINARY_DIR}/validate-dirty-db.py COPYONLY)
 
 #To run plugin_test with all log from blockchain displayed, put --verbose after --, i.e. plugin_test -- --verbose
 add_test(NAME plugin_test COMMAND plugin_test --report_level=detailed --color_output)
@@ -54,6 +55,7 @@ add_test(NAME distributed-transactions-remote-test COMMAND tests/distributed-tra
 # add_test(NAME restart-scenarios-test_replay COMMAND tests/restart-scenarios-test.py -c replay -p4 -v --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 add_test(NAME restart-scenarios-test_hard_replay COMMAND tests/restart-scenarios-test.py -c hardReplay -p4 -v --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 # TODO: add_test(NAME consensus-validation-malicious-producers COMMAND tests/consensus-validation-malicious-producers.py -w 80 --dump-error-details WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_test(NAME validate_dirty_db_test COMMAND tests/validate-dirty-db.py -v --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 
 if(ENABLE_COVERAGE_TESTING)
 

tests/testUtils.py

@@ -1393,7 +1393,7 @@ class Cluster(object):
     # pylint: disable=too-many-return-statements
     # pylint: disable=too-many-branches
     # pylint: disable=too-many-statements
-    def launch(self, pnodes=1, totalNodes=1, prodCount=1, topo="mesh", delay=1, onlyBios=False, dontKill=False):
+    def launch(self, pnodes=1, totalNodes=1, prodCount=1, topo="mesh", delay=1, onlyBios=False, dontKill=False, dontBootstrap=False):
         """Launch cluster.
         pnodes: producer nodes count
         totalNodes: producer + non-producer nodes count
@@ -1456,6 +1456,10 @@ class Cluster(object):
             Utils.Print("ERROR: Cluster doesn't seem to be in sync. Some nodes missing block 1")
             return False
 
+        if dontBootstrap:
+            Utils.Print("Skipping bootstrap.")
+            return True
+
         Utils.Print("Bootstrap cluster.")
         if not Cluster.bootstrap(totalNodes, prodCount, Cluster.__BiosHost, Cluster.__BiosPort, dontKill, onlyBios):
             Utils.Print("ERROR: Bootstrap failed.")

tests/validate-dirty-db.py

+#!/usr/bin/env python3
+
+import testUtils
+
+import argparse
+import random
+import subprocess
+import time
+import os
+import signal
+
+###############################################################
+# Test for validating the dirty db flag sticks repeated nodeos restart attempts
+###############################################################
+
+
+Print=testUtils.Utils.Print
+
+def errorExit(msg="", errorCode=1):
+    Print("ERROR:", msg)
+    exit(errorCode)
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-v", help="verbose logging", action='store_true')
+parser.add_argument("--dont-kill", help="Leave cluster running after test finishes", action='store_true')
+parser.add_argument("--dump-error-details",
+                    help="Upon error print etc/eosio/node_*/config.ini and var/lib/node_*/stderr.log to stdout",
+                    action='store_true')
+parser.add_argument("--keep-logs", help="Don't delete var/lib/node_* folders upon test completion",
+                    action='store_true')
+
+args = parser.parse_args()
+debug=args.v
+pnodes=1
+topo="mesh"
+delay=1
+chainSyncStrategyStr=testUtils.Utils.SyncResyncTag
+total_nodes = pnodes
+killCount=1
+killSignal=testUtils.Utils.SigKillTag
+
+killEosInstances= not args.dont_kill
+dumpErrorDetails=args.dump_error_details
+keepLogs=args.keep_logs
+
+seed=1
+testUtils.Utils.Debug=debug
+testSuccessful=False
+
+random.seed(seed) # Use a fixed seed for repeatability.
+cluster=testUtils.Cluster(walletd=True)
+
+try:
+    cluster.setChainStrategy(chainSyncStrategyStr)
+
+    cluster.killall()
+    cluster.cleanup()
+
+    Print ("producing nodes: %d, topology: %s, delay between nodes launch(seconds): %d, chain sync strategy: %s" % (
+        pnodes, topo, delay, chainSyncStrategyStr))
+
+    Print("Stand up cluster")
+    if cluster.launch(pnodes, total_nodes, topo=topo, delay=delay, dontBootstrap=True) is False:
+        errorExit("Failed to stand up eos cluster.")
+
+    node=cluster.getNode(0)
+    if node is None:
+        errorExit("Cluster in bad state, received None node")
+
+    Print("Kill cluster nodes.")
+    cluster.killall()
+
+    def runNodeosAndGetOutput(nodeId, waitTime=3):
+        """Startup nodeos, wait for waitTime (before forced shutdown) and collect output. Stdout, stderr and return code are returned in a dictionary.
+        Large waittimes have potential to deadlock as the popen PIPE buffer can get filled up."""
+        Print("Launching nodeos process id: %d" % (nodeId))
+        dataDir="var/lib/node_%02d" % (nodeId)
+        cmd="programs/nodeos/nodeos --config-dir etc/eosio/node_bios --data-dir var/lib/node_bios"
+        Print("cmd: %s" % (cmd))
+        proc=subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+        try:
+            proc.wait(waitTime)
+        except (subprocess.TimeoutExpired) as _:
+            Print("ERROR: Nodeos is running beyond the defined wait time. Hard killing nodeos instance.")
+            proc.send_signal(signal.SIGKILL)
+            return (False, None)
+
+        output={}
+        output["stdout"] = proc.stdout.read().decode("utf-8")
+        output["stderr"] = proc.stderr.read().decode("utf-8")
+        output["returncode"] = proc.returncode
+        return (True, output)
+    
+    Print("Restart nodeos repeatedly to ensure dirty database flag sticks.")
+    nodeId=0
+    waitTime=3
+    
+    for i in range(0,3):
+        Print("Attempt %d." % (i))
+        ret = runNodeosAndGetOutput(nodeId, waitTime)
+        if not ret or not ret[0]:
+            exit(1)
+
+        #Print(ret)
+
+        stderr=ret[1]["stderr"]
+        retCode=ret[1]["returncode"]
+        assert(retCode == 2)
+        assert("database dirty flag set" in stderr)
+
+    testSuccessful=True
+finally:
+    if testSuccessful:
+        Print("Test succeeded.")
+    else:
+        Print("Test failed.")
+
+    if not testSuccessful and dumpErrorDetails:
+        cluster.dumpErrorDetails()
+        Print("== Errors see above ==")
+
+    if killEosInstances:
+        Print("Shut down the cluster.")
+        cluster.killall()
+        if testSuccessful and not keepLogs:
+            Print("Cleanup cluster data.")
+            cluster.cleanup()
+
+exit(0)