Refactored crash_gen tool with stronger typing

tests/pytest/crash_gen/__init__.py

 # Helpful Ref: https://stackoverflow.com/questions/24100558/how-can-i-split-a-module-into-multiple-files-without-breaking-a-backwards-compa/24100645
 from crash_gen.service_manager import ServiceManager, TdeInstance, TdeSubProcess
-from crash_gen.misc import Logging, Status, CrashGenError, Dice, Helper, Progress
-from crash_gen.db import DbConn, MyTDSql, DbConnNative, DbManager
-from crash_gen.settings import Settings
-from crash_gen.types import DirPath
\ No newline at end of file

tests/pytest/crash_gen/service_manager.py

@@ -19,10 +19,15 @@ except:
     sys.exit(-1)
 from queue import Queue, Empty
 
-from crash_gen.misc import CrashGenError, Dice, Helper, Logging, Progress, Status
-from crash_gen.db import DbConn, DbTarget
-from crash_gen.settings import Settings
-from crash_gen.types import DirPath
+from .shared.config import Config
+from .shared.db import DbTarget, DbConn
+from .shared.misc import Logging, Helper, CrashGenError, Status, Progress, Dice
+from .shared.types import DirPath
+
+# from crash_gen.misc import CrashGenError, Dice, Helper, Logging, Progress, Status
+# from crash_gen.db import DbConn, DbTarget
+# from crash_gen.settings import Config
+# from crash_gen.types import DirPath
 
 class TdeInstance():
     """
@@ -173,7 +178,7 @@ quorum 2
 
     def getServiceCmdLine(self): # to start the instance
         cmdLine = []
-        if Settings.getConfig().track_memory_leaks:
+        if Config.getConfig().track_memory_leaks:
             Logging.info("Invoking VALGRIND on service...")
             cmdLine = ['valgrind', '--leak-check=yes']
         # TODO: move "exec -c" into Popen(), we can both "use shell" and NOT fork so ask to lose kill control
@@ -789,22 +794,10 @@ class ServiceManagerThread:
     def stop(self):
         # can be called from both main thread or signal handler
 
-        # Linux will send Control-C generated SIGINT to the TDengine process
-        # already, ref:
+        # Linux will send Control-C generated SIGINT to the TDengine process already, ref:
         # https://unix.stackexchange.com/questions/176235/fork-and-how-signals-are-delivered-to-processes
-        # if not self._tdeSubProcess:
-        #     raise RuntimeError("sub process object missing")
-
-        # self._status.set(Status.STATUS_STOPPING)
-        # TdeSubProcess.stop(self._tdeSubProcess) # must stop, no matter what
-        # self._tdeSubProcess = None
-        # if not self._tdeSubProcess.stop(): # everything withing
-        #     if self._tdeSubProcess.isRunning():  # still running, should now never happen
-        #         Logging.error("FAILED to stop sub process, it is still running... pid = {}".format(
-        #             self._tdeSubProcess.getPid()))
-        #     else:
-        #         self._tdeSubProcess = None  # not running any more
-        self.join()  # stop the thread, change the status, etc.
+
+        self.join()  # stop the thread, status change moved to TdeSubProcess
 
         # Check if it's really stopped
         outputLines = 10 # for last output

tests/pytest/crash_gen/settings.py → tests/pytest/crash_gen/shared/config.py

 from __future__ import annotations
 import argparse
+
 from typing import Optional
 
-from crash_gen.misc import CrashGenError
+from .misc import CrashGenError
+
+# from crash_gen.misc import CrashGenError
 
 # gConfig:    Optional[argparse.Namespace]
 
-class Settings:
+class Config:
     _config = None # type Optional[argparse.Namespace]
 
     @classmethod    
-    def init(cls):
-        cls._config = None
+    def init(cls, parser: argparse.ArgumentParser):
+        if cls._config is not None:
+            raise CrashGenError("Config can only be initialized once")
+        cls._config = parser.parse_args()
+        # print(cls._config)
 
     @classmethod
     def setConfig(cls, config: argparse.Namespace):
@@ -26,4 +32,11 @@ class Settings:
 
     @classmethod
     def clearConfig(cls):
-        cls._config = None
\ No newline at end of file
+        cls._config = None
+
+    @classmethod
+    def isSet(cls, cfgKey):
+        cfg = cls.getConfig()
+        if cfgKey not in cfg:
+            return False
+        return cfg.__getattribute__(cfgKey)
\ No newline at end of file

tests/pytest/crash_gen/db.py → tests/pytest/crash_gen/shared/db.py

 from __future__ import annotations
 
 import sys
+import os
+import datetime
 import time
 import threading
 import requests
 from requests.auth import HTTPBasicAuth
-from crash_gen.types import QueryResult
+
 
 import taos
 from util.sql import *
@@ -13,13 +15,12 @@ from util.cases import *
 from util.dnodes import *
 from util.log import *
 
-from .misc import Logging, CrashGenError, Helper, Dice
-import os
-import datetime
 import traceback
 # from .service_manager import TdeInstance
 
-from crash_gen.settings import Settings
+from .config import Config
+from .misc import Logging, CrashGenError, Helper
+from .types import QueryResult
 
 class DbConn:
     TYPE_NATIVE = "native-c"
@@ -250,7 +251,13 @@ class MyTDSql:
     def _execInternal(self, sql):        
         startTime = time.time() 
         # Logging.debug("Executing SQL: " + sql)
+        # ret = None # TODO: use strong type here
+        # try: # Let's not capture the error, and let taos.error.ProgrammingError pass through
         ret = self._cursor.execute(sql)
+        # except taos.error.ProgrammingError as err:
+        #     Logging.warning("Taos SQL execution error: {}, SQL: {}".format(err.msg, sql))
+        #     raise CrashGenError(err.msg)
+            
         # print("\nSQL success: {}".format(sql))
         queryTime =  time.time() - startTime
         # Record the query time
@@ -262,7 +269,7 @@ class MyTDSql:
                 cls.lqStartTime = startTime
 
         # Now write to the shadow database
-        if Settings.getConfig().use_shadow_db:
+        if Config.isSet('use_shadow_db'):
             if sql[:11] == "INSERT INTO":
                 if sql[:16] == "INSERT INTO db_0":
                     sql2 = "INSERT INTO db_s" + sql[16:]

tests/pytest/crash_gen/misc.py → tests/pytest/crash_gen/shared/misc.py

@@ -47,7 +47,7 @@ class Logging:
         return cls.logger
 
     @classmethod
-    def clsInit(cls, gConfig): # TODO: refactor away gConfig
+    def clsInit(cls, debugMode: bool):
         if cls.logger:
             return
         
@@ -62,12 +62,8 @@ class Logging:
         # print("setting logger variable")
         # global logger
         cls.logger = MyLoggingAdapter(_logger, {})
-
-        if (gConfig.debug):
-            cls.logger.setLevel(logging.DEBUG)  # default seems to be INFO
-        else:
-            cls.logger.setLevel(logging.INFO)
-
+        cls.logger.setLevel(logging.DEBUG if debugMode else logging.INFO)  # default seems to be INFO
+        
     @classmethod
     def info(cls, msg):
         cls.logger.info(msg)

tests/pytest/crash_gen/shared/types.py

+from typing import Any, List, Dict, NewType
+from enum import Enum
+
+DirPath = NewType('DirPath', str)
+
+QueryResult = NewType('QueryResult', List[List[Any]])
+
+class TdDataType(Enum):
+    '''
+    Use a Python Enum types of represent all the data types in TDengine.
+
+    Ref: https://www.taosdata.com/cn/documentation/taos-sql#data-type
+    '''
+    TIMESTAMP = 'TIMESTAMP'
+    INT       = 'INT'
+    BIGINT    = 'BIGINT'
+    FLOAT     = 'FLOAT'
+    DOUBLE    = 'DOUBLE'
+    BINARY    = 'BINARY'
+    BINARY16  = 'BINARY(16)'  # TODO: get rid of this hack
+    BINARY200 = 'BINARY(200)'
+    SMALLINT  = 'SMALLINT'
+    TINYINT   = 'TINYINT'
+    BOOL      = 'BOOL'
+    NCHAR     = 'NCHAR'
+
+TdColumns = Dict[str, TdDataType]
+TdTags    = Dict[str, TdDataType]

tests/pytest/crash_gen/types.py

-from typing import Any, List, NewType
-
-DirPath = NewType('DirPath', str)
-
-QueryResult = NewType('QueryResult', List[List[Any]])
\ No newline at end of file