Merge branch 'develop' into hotfix/mergetodev1

documentation20/webdocs/markdowndocs/faq-ch.md

@@ -23,14 +23,18 @@
 
 客户端遇到链接故障，请按照下面的步骤进行检查：
 
-1. 确保客户端与服务端版本号是完全一致的，开源社区版和企业版也不能混用
-2. 在服务器，执行 `systemctl status taosd` 检查*taosd*运行状态。如果没有运行，启动*taosd*
-3. 确认客户端连接时指定了正确的服务器FQDN (Fully Qualified Domain Name(可在服务器上执行Linux命令hostname -f获得）
-4. ping服务器FQDN，如果没有反应，请检查你的网络，DNS设置，或客户端所在计算机的系统hosts文件
-5. 检查防火墙设置，确认TCP/UDP 端口6030-6039 是打开的
-6. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/lib/taos*里, 并且*/usr/local/lib/taos*在系统库函数搜索路径*LD_LIBRARY_PATH*里 
-7. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*driver/c/taos.dll*在你的系统搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*)
-8. 如果仍不能排除连接故障，请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅
+1. 检查网络环境
+    * 云服务器：检查云服务器的安全组是否打开TCP/UDP 端口6030-6039的访问权限
+    * 本地虚拟机：检查网络能否ping通，尽量避免使用`localhost` 作为hostname
+    * 公司服务器：如果为NAT网络环境，请务必检查服务器能否将消息返回值客户端
+2. 确保客户端与服务端版本号是完全一致的，开源社区版和企业版也不能混用
+3. 在服务器，执行 `systemctl status taosd` 检查*taosd*运行状态。如果没有运行，启动*taosd*
+4. 确认客户端连接时指定了正确的服务器FQDN (Fully Qualified Domain Name(可在服务器上执行Linux命令hostname -f获得）
+5. ping服务器FQDN，如果没有反应，请检查你的网络，DNS设置，或客户端所在计算机的系统hosts文件
+6. 检查防火墙设置，确认TCP/UDP 端口6030-6039 是打开的
+7. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/lib/taos*里, 并且*/usr/local/lib/taos*在系统库函数搜索路径*LD_LIBRARY_PATH*里 
+8. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*driver/c/taos.dll*在你的系统搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*)
+9. 如果仍不能排除连接故障，请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅
    检查UDP端口连接是否工作：`nc -vuz {hostIP} {port} `
    检查服务器侧TCP端口连接是否工作：`nc -l {port}`
    检查客户端侧TCP端口链接是否工作：`nc {hostIP} {port}`

src/connector/jdbc/src/test/java/TestPreparedStatement.java

@@ -12,7 +12,6 @@ public class TestPreparedStatement {
             Class.forName("com.taosdata.jdbc.TSDBDriver");
             Properties properties = new Properties();
             properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, "localhost");
-
             connection = DriverManager.getConnection("jdbc:TAOS://localhost:0/", properties);
             String rawSql = "select * from test.log0601";
 //            String[] params = new String[]{"ts", "c1"};

src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java

@@ -13,7 +13,6 @@ public class TestTSDBDatabaseMetaData {
             Class.forName("com.taosdata.jdbc.TSDBDriver");
             Properties properties = new Properties();
             properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, "localhost");
-
             connection = DriverManager.getConnection("jdbc:TAOS://localhost:0/", properties);
             dbMetaData = connection.getMetaData();
             resSet = dbMetaData.getCatalogs();

src/connector/jdbc/src/test/java/TestTSDBSubscribe.java

@@ -3,7 +3,6 @@ import com.taosdata.jdbc.TSDBDriver;
 import com.taosdata.jdbc.TSDBResultSet;
 import com.taosdata.jdbc.TSDBSubscribe;
 
-import java.sql.Connection;
 import java.sql.DriverManager;
 import java.util.Properties;
 

src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java

@@ -40,7 +40,6 @@ public class BatchInsertTest extends BaseTest {
         properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
-
         connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties);
                 
         statement = connection.createStatement();

src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java

@@ -29,7 +29,6 @@ public class ConnectionTest extends BaseTest {
         properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
-
         connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties);
 
         assertTrue(null != connection);

src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java

@@ -26,8 +26,7 @@ public class DatabaseMetaDataTest extends BaseTest {
         properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
-
-        connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" , properties);
+        connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties);
 
         String sql = "drop database if exists " + dbName;
         statement = (TSDBPreparedStatement) connection.prepareStatement(sql);

src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java

@@ -28,7 +28,6 @@ public class ImportTest extends BaseTest {
         properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
-
         connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties);
 
         statement = connection.createStatement();

src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java

@@ -33,7 +33,6 @@ public class PreparedStatementTest extends BaseTest {
         properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
-
         connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties);
 
         String sql = "drop database if exists " + dbName;

src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java

@@ -28,8 +28,7 @@ public class SelectTest extends BaseTest {
         properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
-
-        connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" , properties);
+        connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties);
 
         statement = connection.createStatement();
         statement.executeUpdate("drop database if exists " + dbName);

src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java

@@ -31,8 +31,7 @@ public class StableTest extends BaseTest {
         properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
-
-        connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" , properties);
+        connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties);
 
         statement = connection.createStatement();
         statement.executeUpdate("create database if not exists " + dbName);

src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java

@@ -30,7 +30,6 @@ public class StatementTest extends BaseTest {
         properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
-
         connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties);
 
         statement = connection.createStatement();

src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java

@@ -32,7 +32,6 @@ public class SubscribeTest extends BaseTest {
         properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
         properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
-
         connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties);
 
         statement = connection.createStatement();

src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java

@@ -12,7 +12,7 @@ public class TSDBDriverTest {
     @Test
     public void urlParserTest() throws SQLException {
         TSDBDriver driver = new TSDBDriver();
-        String url = "jdbc:TSDB://127.0.0.1:0/db?user=root&password=your_password";
+        String url = "jdbc:TSDB://127.0.0.1:0/db";
 
         Properties properties = new Properties(); 
         driver.parseURL(url, properties);

src/mnode/src/mnodeShow.c

@@ -313,6 +313,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) {
     if (pDb->status != TSDB_DB_STATUS_READY) {
       mError("db:%s, status:%d, in dropping", pDb->name, pDb->status);
       code = TSDB_CODE_MND_DB_IN_DROPPING;
+      mnodeDecDbRef(pDb);
       goto connect_over;
     }
     mnodeDecDbRef(pDb);

src/mnode/src/mnodeTable.c

@@ -294,6 +294,7 @@ static int32_t mnodeChildTableActionRestored() {
       SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb};
       sdbDeleteRow(&desc);
       mnodeDecTableRef(pTable);
+      mnodeDecDbRef(pDb);
       continue;
     }
     mnodeDecDbRef(pDb);
@@ -1259,6 +1260,7 @@ static int32_t mnodeGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow,
     
   if (pDb->status != TSDB_DB_STATUS_READY) {
     mError("db:%s, status:%d, in dropping", pDb->name, pDb->status);
+    mnodeDecDbRef(pDb);
     return TSDB_CODE_MND_DB_IN_DROPPING;
   }
 
@@ -1323,6 +1325,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows,
   
   if (pDb->status != TSDB_DB_STATUS_READY) {
     mError("db:%s, status:%d, in dropping", pDb->name, pDb->status);
+    mnodeDecDbRef(pDb);
     return 0;
   }
 
@@ -2495,6 +2498,7 @@ static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void
   
   if (pDb->status != TSDB_DB_STATUS_READY) {
     mError("db:%s, status:%d, in dropping", pDb->name, pDb->status);
+    mnodeDecDbRef(pDb);
     return TSDB_CODE_MND_DB_IN_DROPPING;
   }
 
@@ -2548,6 +2552,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
   
   if (pDb->status != TSDB_DB_STATUS_READY) {
     mError("db:%s, status:%d, in dropping", pDb->name, pDb->status);
+    mnodeDecDbRef(pDb);
     return 0;
   }
 
@@ -2716,6 +2721,7 @@ static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, vo
   
   if (pDb->status != TSDB_DB_STATUS_READY) {
     mError("db:%s, status:%d, in dropping", pDb->name, pDb->status);
+    mnodeDecDbRef(pDb);
     return TSDB_CODE_MND_DB_IN_DROPPING;
   }
 
@@ -2768,6 +2774,7 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro
   
   if (pDb->status != TSDB_DB_STATUS_READY) {
     mError("db:%s, status:%d, in dropping", pDb->name, pDb->status);
+    mnodeDecDbRef(pDb);
     return 0;
   }
   

src/mnode/src/mnodeVgroup.c

@@ -89,6 +89,7 @@ static int32_t mnodeVgroupActionInsert(SSdbOper *pOper) {
   
   if (pDb->status != TSDB_DB_STATUS_READY) {
     mError("vgId:%d, db:%s status:%d, in dropping", pVgroup->vgId, pDb->name, pDb->status);
+    mnodeDecDbRef(pDb);
     return TSDB_CODE_MND_DB_IN_DROPPING;
   }
 
@@ -617,6 +618,7 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p
   
   if (pDb->status != TSDB_DB_STATUS_READY) {
     mError("db:%s, status:%d, in dropping", pDb->name, pDb->status);
+    mnodeDecDbRef(pDb);
     return TSDB_CODE_MND_DB_IN_DROPPING;
   }
 
@@ -708,6 +710,7 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v
 
   if (pDb->status != TSDB_DB_STATUS_READY) {
     mError("db:%s, status:%d, in dropping", pDb->name, pDb->status);
+    mnodeDecDbRef(pDb);
     return 0;
   }
 
@@ -784,7 +787,10 @@ void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
   if (pTable->sid >= 1) {
     taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid);
     pVgroup->numOfTables++;
-    mnodeIncVgroupRef(pVgroup);
+    // The create vgroup message may be received later than the create table message
+    // and the writing order in sdb is therefore uncertain
+    // which will cause the reference count of the vgroup to be incorrect when restarting
+    // mnodeIncVgroupRef(pVgroup);
   }
 }
 
@@ -792,7 +798,10 @@ void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
   if (pTable->sid >= 1) {
     taosFreeId(pVgroup->idPool, pTable->sid);
     pVgroup->numOfTables--;
-    mnodeDecVgroupRef(pVgroup);
+    // The create vgroup message may be received later than the create table message
+    // and the writing order in sdb is therefore uncertain
+    // which will cause the reference count of the vgroup to be incorrect when restarting
+    // mnodeDecVgroupRef(pVgroup);
   }
 }
 

src/util/src/hash.c

@@ -377,6 +377,12 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe
     }
   }
 
+  if (pe->num == 0) {
+    assert(pe->next == NULL);
+  } else {
+    assert(pe->next != NULL);
+  }
+
   if (pHashObj->type == HASH_ENTRY_LOCK) {
     taosWUnLockLatch(&pe->latch);
   }
@@ -390,22 +396,8 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe
   if (pRes != NULL) {
     atomic_sub_fetch_64(&pHashObj->size, 1);
     FREE_HASH_NODE(pHashObj, pRes);
-
-    if (pe->num == 0) {
-      assert(pe->next == NULL);
-    } else {
-      assert(pe->next != NULL);
-    }
-
     return 0;
   } else {
-
-    if (pe->num == 0) {
-      assert(pe->next == NULL);
-    } else {
-      assert(pe->next != NULL);
-    }
-
     return -1;
   }
 }

tests/pytest/crash_gen.py

@@ -161,6 +161,21 @@ class WorkerThread:
                 logger.debug("[TRD] Thread Coordinator not running any more, worker thread now stopping...")
                 break
 
+            # Before we fetch the task and run it, let's ensure we properly "use" the database
+            try:
+                if (gConfig.per_thread_db_connection):  # most likely TRUE
+                    if not self._dbConn.isOpen:  # might have been closed during server auto-restart
+                        self._dbConn.open()
+                self.useDb() # might encounter exceptions. TODO: catch
+            except taos.error.ProgrammingError as err:
+                errno = Helper.convertErrno(err.errno)
+                if errno in [0x383, 0x386, 0x00B, 0x014]  : # invalid database, dropping, Unable to establish connection, Database not ready
+                    # ignore
+                    dummy = 0
+                else:
+                    print("\nCaught programming error. errno=0x{:X}, msg={} ".format(errno, err.msg))
+                    raise
+
             # Fetch a task from the Thread Coordinator
             logger.debug( "[TRD] Worker thread [{}] about to fetch task".format(self._tid))
             task = tc.fetchTask()
@@ -324,10 +339,12 @@ class ThreadCoordinator:
             logger.debug("[STT] transition ended")
             # Due to limitation (or maybe not) of the Python library,
             # we cannot share connections across threads
-            if sm.hasDatabase():
-                for t in self._pool.threadList:
-                    logger.debug("[DB] use db for all worker threads")
-                    t.useDb()
+            # Here we are in main thread, we cannot operate the connections created in workers
+            # Moving below to task loop
+            # if sm.hasDatabase():
+            #     for t in self._pool.threadList:
+            #         logger.debug("[DB] use db for all worker threads")
+            #         t.useDb()
                     # t.execSql("use db") # main thread executing "use
                     # db" on behalf of every worker thread
         except taos.error.ProgrammingError as err:
@@ -387,7 +404,7 @@ class ThreadCoordinator:
                 transitionFailed = self._doTransition() # To start, we end step -1 first
             except taos.error.ProgrammingError as err:
                 transitionFailed = True
-                errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno  # correct error scheme
+                errno2 = Helper.convertErrno(err.errno)  # correct error scheme
                 errMsg = "Transition failed: errno=0x{:X}, msg: {}".format(errno2, err)
                 logger.info(errMsg)
                 self._execStats.registerFailure(errMsg)
@@ -468,6 +485,10 @@ class ThreadCoordinator:
 
 # We define a class to run a number of threads in locking steps.
 
+class Helper:
+    @classmethod
+    def convertErrno(cls, errno):
+        return errno if (errno > 0) else 0x80000000 + errno
 
 class ThreadPool:
     def __init__(self, numThreads, maxSteps):
@@ -613,8 +634,7 @@ class DbConn:
 
     def resetDb(self):  # reset the whole database, etc.
         if (not self.isOpen):
-            raise RuntimeError(
-                "Cannot reset database until connection is open")
+            raise RuntimeError("Cannot reset database until connection is open")
         # self._tdSql.prepare() # Recreate database, etc.
 
         self.execute('drop database if exists db')
@@ -681,8 +701,7 @@ class DbConnRest(DbConn):
 
     def close(self):
         if (not self.isOpen):
-            raise RuntimeError(
-                "Cannot clean up database until connection is open")
+            raise RuntimeError("Cannot clean up database until connection is open")
         # Do nothing for REST
         logger.debug("[DB] REST Database connection closed")
         self.isOpen = False
@@ -747,27 +766,32 @@ class DbConnRest(DbConn):
 
 
 class MyTDSql:
-    def __init__(self):
+    def __init__(self, hostAddr, cfgPath):
+        # Make the DB connection
+        self._conn = taos.connect(host=hostAddr, config=cfgPath) 
+        self._cursor = self._conn.cursor()
+
         self.queryRows = 0
         self.queryCols = 0
         self.affectedRows = 0
 
-    def init(self, cursor, log=True):
-        self.cursor = cursor
+    # def init(self, cursor, log=True):
+    #     self.cursor = cursor
         # if (log):
         #     caller = inspect.getframeinfo(inspect.stack()[1][0])
         #     self.cursor.log(caller.filename + ".sql")
 
     def close(self):
-        self.cursor.close()
+        self._conn.close() # TODO: very important, cursor close does NOT close DB connection!
+        self._cursor.close()
 
     def query(self, sql):
         self.sql = sql
         try:
-            self.cursor.execute(sql)
-            self.queryResult = self.cursor.fetchall()
+            self._cursor.execute(sql)
+            self.queryResult = self._cursor.fetchall()
             self.queryRows = len(self.queryResult)
-            self.queryCols = len(self.cursor.description)
+            self.queryCols = len(self._cursor.description)
         except Exception as e:
             # caller = inspect.getframeinfo(inspect.stack()[1][0])
             # args = (caller.filename, caller.lineno, sql, repr(e))
@@ -778,7 +802,7 @@ class MyTDSql:
     def execute(self, sql):
         self.sql = sql
         try:
-            self.affectedRows = self.cursor.execute(sql)
+            self.affectedRows = self._cursor.execute(sql)
         except Exception as e:
             # caller = inspect.getframeinfo(inspect.stack()[1][0])
             # args = (caller.filename, caller.lineno, sql, repr(e))
@@ -791,13 +815,13 @@ class DbConnNative(DbConn):
     # Class variables
     _lock = threading.Lock()
     _connInfoDisplayed = False
+    totalConnections = 0 # Not private
 
     def __init__(self):
         super().__init__()
         self._type = self.TYPE_NATIVE
         self._conn = None
-        self._cursor = None
-        
+        # self._cursor = None        
 
     def getBuildPath(self):
         selfPath = os.path.dirname(os.path.realpath(__file__))
@@ -814,7 +838,8 @@ class DbConnNative(DbConn):
                     buildPath = root[:len(root) - len("/build/bin")]
                     break
         if buildPath == None:
-            raise RuntimeError("Failed to determine buildPath, selfPath={}".format(selfPath))
+            raise RuntimeError("Failed to determine buildPath, selfPath={}, projPath={}"
+                .format(selfPath, projPath))
         return buildPath
 
     
@@ -822,33 +847,40 @@ class DbConnNative(DbConn):
         cfgPath = self.getBuildPath() + "/test/cfg"
         hostAddr = "127.0.0.1"
 
-        with self._lock: # force single threading for opening DB connections
-            if not self._connInfoDisplayed:
-                self.__class__._connInfoDisplayed = True # updating CLASS variable
-                logger.info("Initiating TAOS native connection to {}, using config at {}".format(hostAddr, cfgPath))
-            
-            self._conn = taos.connect(host=hostAddr, config=cfgPath)  # TODO: make configurable
-            self._cursor = self._conn.cursor()
+        cls = self.__class__ # Get the class, to access class variables
+        with cls._lock: # force single threading for opening DB connections. # TODO: whaaat??!!!
+            if not cls._connInfoDisplayed:
+                cls._connInfoDisplayed = True # updating CLASS variable
+                logger.info("Initiating TAOS native connection to {}, using config at {}".format(hostAddr, cfgPath))                    
+            # Make the connection         
+            # self._conn = taos.connect(host=hostAddr, config=cfgPath)  # TODO: make configurable
+            # self._cursor = self._conn.cursor()
+            # Record the count in the class
+            self._tdSql = MyTDSql(hostAddr, cfgPath) # making DB connection
+            cls.totalConnections += 1 
         
-        self._cursor.execute('reset query cache')
+        self._tdSql.execute('reset query cache')
         # self._cursor.execute('use db') # do this at the beginning of every
 
         # Open connection
-        self._tdSql = MyTDSql()
-        self._tdSql.init(self._cursor)
-
+        # self._tdSql = MyTDSql()
+        # self._tdSql.init(self._cursor)
+        
     def close(self):
         if (not self.isOpen):
-            raise RuntimeError(
-                "Cannot clean up database until connection is open")
+            raise RuntimeError("Cannot clean up database until connection is open")
         self._tdSql.close()
+        # Decrement the class wide counter
+        cls = self.__class__ # Get the class, to access class variables
+        with cls._lock:
+            cls.totalConnections -= 1
+
         logger.debug("[DB] Database connection closed")
         self.isOpen = False
 
     def execute(self, sql):
         if (not self.isOpen):
-            raise RuntimeError(
-                "Cannot execute database commands until connection is open")
+            raise RuntimeError("Cannot execute database commands until connection is open")
         logger.debug("[SQL] Executing SQL: {}".format(sql))
         self._lastSql = sql
         nRows = self._tdSql.execute(sql)
@@ -1528,7 +1560,7 @@ class Task():
         try:
             self._executeInternal(te, wt)  # TODO: no return value?
         except taos.error.ProgrammingError as err:
-            errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno  # correct error scheme
+            errno2 = Helper.convertErrno(err.errno)
             if (gConfig.continue_on_exception):  # user choose to continue
                 self.logDebug("[=] Continue after TAOS exception: errno=0x{:X}, msg: {}, SQL: {}".format(
                         errno2, err, wt.getDbConn().getLastSql()))
@@ -1678,9 +1710,8 @@ class ExecutionStats:
         logger.info(
             "| Total Elapsed Time (from wall clock): {:.3f} seconds".format(
                 self._elapsedTime))
-        logger.info(
-            "| Top numbers written: {}".format(
-                TaskExecutor.getBoundedList()))
+        logger.info("| Top numbers written: {}".format(TaskExecutor.getBoundedList()))
+        logger.info("| Total Number of Active DB Native Connections: {}".format(DbConnNative.totalConnections))
         logger.info(
             "----------------------------------------------------------------------")
 
@@ -1789,7 +1820,7 @@ class TdSuperTable:
         try:
             dbc.query("select TBNAME from db.{}".format(self._stName))  # TODO: analyze result set later            
         except taos.error.ProgrammingError as err:                    
-            errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno
+            errno2 = Helper.convertErrno(err.errno) 
             logger.debug("[=] Failed to get tables from super table: errno=0x{:X}, msg: {}".format(errno2, err))
             raise
 
@@ -1891,7 +1922,7 @@ class TaskReadData(StateTransitionTask):
                 if aggExpr not in ['stddev(speed)']: #TODO: STDDEV not valid for super tables?!
                     dbc.execute("select {} from db.{}".format(aggExpr, sTable.getName()))
             except taos.error.ProgrammingError as err:                    
-                errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno
+                errno2 = Helper.convertErrno(err.errno)
                 logger.debug("[=] Read Failure: errno=0x{:X}, msg: {}, SQL: {}".format(errno2, err, dbc.getLastSql()))
                 raise
 
@@ -1920,9 +1951,8 @@ class TaskDropSuperTable(StateTransitionTask):
                     self.execWtSql(wt, "drop table {}".format(
                         regTableName))  # nRows always 0, like MySQL
                 except taos.error.ProgrammingError as err:
-                    # correcting for strange error number scheme
-                    errno2 = err.errno if (
-                        err.errno > 0) else 0x80000000 + err.errno
+                    # correcting for strange error number scheme                    
+                    errno2 = Helper.convertErrno(err.errno)
                     if (errno2 in [0x362]):  # mnode invalid table name
                         isSuccess = False
                         logger.debug(
@@ -2429,7 +2459,11 @@ class ServiceManagerThread:
         for line in iter(out.readline, b''):
             # print("Finished reading a line: {}".format(line))
             # print("Adding item to queue...")
-            line = line.decode("utf-8").rstrip()
+            try:
+                line = line.decode("utf-8").rstrip()
+            except UnicodeError:
+                print("\nNon-UTF8 server output: {}\n".format(line))
+
             # This might block, and then causing "out" buffer to block
             queue.put(line)
             self._printProgress("_i")
@@ -2455,7 +2489,7 @@ class ServiceManagerThread:
 
     def svcErrorReader(self, err: IO, queue):
         for line in iter(err.readline, b''):
-            print("\nTD Svc STDERR: {}".format(line))
+            print("\nTDengine Service (taosd) ERROR (from stderr): {}".format(line))
 
 
 class TdeSubProcess:

tests/pytest/insert/nchar.py

@@ -35,6 +35,8 @@ class TDTestCase:
         tdSql.checkRows(2)
         tdSql.checkData(1, 1, '涛思数据')
 
+        tdSql.error("insert into tb values (now, 'taosdata001')")
+        
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

tests/pytest/tag_lite/bigint.py

@@ -575,6 +575,20 @@ class TDTestCase:
         # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT
 # convert end
 
+        tdSql.execute("create database db")
+        tdSql.execute("use db")
+        tdSql.execute(
+            "create table if not exists st (ts timestamp, tagtype int) tags(dev bigint)")
+                
+        tdSql.execute(
+            'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 63) - 1))
+        tdSql.execute(
+            'CREATE TABLE if not exists dev_002 using st tags(%d)' % (-1 * pow(2, 63) + 1))
+
+        print("==============step2")    
+        tdSql.query("show tables")
+        tdSql.checkRows(2)   
+
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

tests/pytest/tag_lite/binary.py

@@ -579,6 +579,20 @@ class TDTestCase:
         # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT
 # convert end
 
+        tdSql.execute("create database db")
+        tdSql.execute("use db")
+        tdSql.execute(
+            "create table if not exists st (ts timestamp, tagtype int) tags(dev binary(5))")
+        tdSql.error(
+            'CREATE TABLE if not exists dev_001 using st tags("dev_001")')
+        
+        tdSql.execute(
+            'CREATE TABLE if not exists dev_002 using st tags("dev")')
+
+        print("==============step2")    
+        tdSql.query("show tables")
+        tdSql.checkRows(1)        
+
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

tests/pytest/tag_lite/float.py

@@ -575,6 +575,23 @@ class TDTestCase:
         # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT
 # convert end
 
+        tdSql.execute("create database db")
+        tdSql.execute("use db")
+        tdSql.execute(
+            "create table if not exists st (ts timestamp, tagtype int) tags(dev float)")
+        tdSql.error(
+            'CREATE TABLE if not exists dev_001 using st tags(%f)' % -3.4E38 - 1)
+        tdSql.error(
+            'CREATE TABLE if not exists dev_001 using st tags(%f)' % 3.4E38 + 1)
+        
+        tdSql.execute(
+            'CREATE TABLE if not exists dev_001 using st tags(%f)' % 3.4E38)
+        tdSql.execute(
+            'CREATE TABLE if not exists dev_002 using st tags(%f)' % -3.4E38)
+          
+        tdSql.query("show tables")
+        tdSql.checkRows(2)  
+
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

tests/pytest/tag_lite/int.py

@@ -574,6 +574,24 @@ class TDTestCase:
         # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT
 # convert end
 
+        tdSql.execute("create database db")
+        tdSql.execute("use db")
+        tdSql.execute(
+            "create table if not exists st (ts timestamp, tagtype int) tags(dev int)")
+        tdSql.error(
+            'CREATE TABLE if not exists dev_001 using st tags(%d)' % pow(2, 31))
+        tdSql.error(
+            'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1 * pow(2, 31)))
+        
+        tdSql.execute(
+            'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 31) - 1))
+        tdSql.execute(
+            'CREATE TABLE if not exists dev_002 using st tags(%d)' % (-1 * pow(2, 31) + 1))
+
+        print("==============step2")    
+        tdSql.query("show tables")
+        tdSql.checkRows(2)   
+
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

tests/pytest/tag_lite/nchar.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import tdLog
+from util.cases import tdCases
+from util.sql import tdSql
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+
+        print("==============step1")
+        tdSql.execute(
+            "create table if not exists st (ts timestamp, tagtype int) tags(dev nchar(5))")
+        tdSql.error(
+            'CREATE TABLE if not exists dev_001 using st tags("dev_001")')
+        tdSql.execute(
+            'CREATE TABLE if not exists dev_002 using st tags("dev")')
+
+
+        print("==============step2")    
+        tdSql.query("show tables")
+        tdSql.checkRows(1)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/tag_lite/smallint.py

@@ -574,6 +574,23 @@ class TDTestCase:
         # TSIM:
         # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT
 # convert end
+        tdSql.execute("create database db")
+        tdSql.execute("use db")
+        tdSql.execute(
+            "create table if not exists st (ts timestamp, tagtype int) tags(dev smallint)")
+        tdSql.error(
+            'CREATE TABLE if not exists dev_001 using st tags(%d)' % pow(2, 15))
+        tdSql.error(
+            'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1 * pow(2, 15)))
+        
+        tdSql.execute(
+            'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 15) - 1))
+        tdSql.execute(
+            'CREATE TABLE if not exists dev_002 using st tags(%d)' % (-1 * pow(2, 15) + 1))
+
+        print("==============step2")    
+        tdSql.query("show tables")
+        tdSql.checkRows(2)   
 
     def stop(self):
         tdSql.close()

tests/pytest/tag_lite/tinyint.py

@@ -575,6 +575,24 @@ class TDTestCase:
         # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT
 # convert end
 
+        tdSql.execute("create database db")
+        tdSql.execute("use db")
+        tdSql.execute(
+            "create table if not exists st (ts timestamp, tagtype int) tags(dev tinyint)")
+        tdSql.error(
+            'CREATE TABLE if not exists dev_001 using st tags(%d)' % pow(2, 7))
+        tdSql.error(
+            'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1 * pow(2, 7)))
+        
+        tdSql.execute(
+            'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 7) - 1))
+        tdSql.execute(
+            'CREATE TABLE if not exists dev_002 using st tags(%d)' % (-1 * pow(2, 7) + 1))
+
+        print("==============step2")    
+        tdSql.query("show tables")
+        tdSql.checkRows(2)   
+
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

tests/script/tmp/182.sim

+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+
+system sh/cfg.sh -n dnode1 -c walLevel -v 2
+system sh/cfg.sh -n dnode2 -c walLevel -v 2
+system sh/cfg.sh -n dnode3 -c walLevel -v 2
+system sh/cfg.sh -n dnode4 -c walLevel -v 2
+
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
+
+system sh/cfg.sh -n dnode1 -c activeCode -v eglxDLzRpslJWl7OxrPZ2K3sQ5631AP9SVpezsaz2dhJWl7OxrPZ2ElaXs7Gs9nYSVpezsaz2djGIj5StnQ3ZvLHcsE8cwcN
+
+system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
+
+system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 100000
+system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 100000
+system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 100000
+system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 100000
+
+system sh/cfg.sh -n dnode1 -c http -v 1
+system sh/cfg.sh -n dnode2 -c http -v 1
+system sh/cfg.sh -n dnode3 -c http -v 1
+system sh/cfg.sh -n dnode4 -c http -v 1
+
+
+system sh/cfg.sh -n dnode1 -c firstEp      -v 127.0.0.1:6030
+system sh/cfg.sh -n dnode1 -c secondEp     -v 127.0.0.1:6030
+system sh/cfg.sh -n dnode1 -c serverPort   -v 6030
+system sh/cfg.sh -n dnode1 -c fqdn         -v 127.0.0.1
+#system sh/exec.sh -n dnode1 -s start