Merge branch 'develop' into feature/query

cmake/install.inc

@@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS)
   #INSTALL(TARGETS taos RUNTIME DESTINATION driver)
   #INSTALL(TARGETS shell RUNTIME DESTINATION .)
   IF (TD_MVN_INSTALLED)
-    INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.8-dist.jar DESTINATION connector/jdbc)
+    INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.13-dist.jar DESTINATION connector/jdbc)
   ENDIF ()
 ELSEIF (TD_DARWIN)
   SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")

cmake/version.inc

@@ -4,7 +4,7 @@ PROJECT(TDengine)
 IF (DEFINED VERNUMBER)
   SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "2.0.7.0")
+  SET(TD_VER_NUMBER "2.0.8.0")
 ENDIF ()
 
 IF (DEFINED VERCOMPATIBLE)

snap/snapcraft.yaml

 name: tdengine
 base: core18
-version: '2.0.7.0'
+version: '2.0.8.0'
 icon: snap/gui/t-dengine.svg
 summary: an open-source big data platform designed and optimized for IoT. 
 description: |
@@ -72,7 +72,7 @@ parts:
       - usr/bin/taosd
       - usr/bin/taos
       - usr/bin/taosdemo
-      - usr/lib/libtaos.so.2.0.6.0
+      - usr/lib/libtaos.so.2.0.8.0
       - usr/lib/libtaos.so.1
       - usr/lib/libtaos.so
 

src/balance/src/balance.c

@@ -162,7 +162,6 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
              pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole);
       mnodeDecDnodeRef(pDnode);
     }
-    sdbFreeIter(pIter);
 
     if (mnodeGetOnlineDnodesNum() == 0) {
       return TSDB_CODE_MND_NOT_READY;
@@ -377,15 +376,13 @@ static bool balanceMonitorBalance() {
                  srcScore, pDestDnode->score, destScore);
           balanceAddVnode(pVgroup, pSrcDnode, pDestDnode);
           mnodeDecVgroupRef(pVgroup);
-          sdbFreeIter(pIter);
+          mnodeCancelGetNextVgroup(pIter);
           return true;
         }
       }
 
       mnodeDecVgroupRef(pVgroup);
     }
-
-    sdbFreeIter(pIter);
   }
 
   return false;
@@ -413,8 +410,6 @@ void balanceReset() {
     mnodeDecDnodeRef(pDnode);
   }
 
-  sdbFreeIter(pIter);
-
   tsAccessSquence = 0;
 }
 
@@ -443,12 +438,11 @@ static int32_t balanceMonitorVgroups() {
 
     mnodeDecVgroupRef(pVgroup);
     if (code == TSDB_CODE_SUCCESS) {
+      mnodeCancelGetNextVgroup(pIter);
       break;
     }
   }
 
-  sdbFreeIter(pIter);
-
   return hasUpdatingVgroup;
 }
 
@@ -465,10 +459,11 @@ static bool balanceMonitorDnodeDropping(SDnodeObj *pDnode) {
     hasThisDnode = balanceCheckDnodeInVgroup(pDnode, pVgroup);
     mnodeDecVgroupRef(pVgroup);
 
-    if (hasThisDnode) break;
+    if (hasThisDnode) {
+      mnodeCancelGetNextVgroup(pIter);
+      break;
+    }
   }
-
-  sdbFreeIter(pIter);
 
   if (!hasThisDnode) {
     mInfo("dnode:%d, dropped for all vnodes are moving to other dnodes", pDnode->dnodeId);
@@ -499,20 +494,18 @@ static bool balanceMontiorDropping() {
       pDnode->status = TAOS_DN_STATUS_DROPPING;
       mnodeUpdateDnode(pDnode);
       mnodeDecDnodeRef(pDnode);
-      sdbFreeIter(pIter);
+      mnodeCancelGetNextDnode(pIter);
       return true;
     }
 
     if (pDnode->status == TAOS_DN_STATUS_DROPPING) {
       bool ret = balanceMonitorDnodeDropping(pDnode);
       mnodeDecDnodeRef(pDnode);
-      sdbFreeIter(pIter);
+      mnodeCancelGetNextDnode(pIter);
       return ret;
     }
   }
 
-  sdbFreeIter(pIter);
-
   return false;
 }
 
@@ -556,8 +549,6 @@ static void balanceSetVgroupOffline(SDnodeObj* pDnode) {
     }
     mnodeDecVgroupRef(pVgroup);
   }
-
-  sdbFreeIter(pIter);
 }
 
 static void balanceCheckDnodeAccess() {
@@ -578,8 +569,6 @@ static void balanceCheckDnodeAccess() {
     }
     mnodeDecDnodeRef(pDnode);
   }
-
-  sdbFreeIter(pIter);
 }
 
 static void balanceProcessBalanceTimer(void *handle, void *tmrId) {
@@ -630,6 +619,7 @@ void balanceAsyncNotify() {
 int32_t balanceInit() {
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_SCORES, balanceGetScoresMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_SCORES, balanceRetrieveScores);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_SCORES, mnodeCancelGetNextDnode);
   
   pthread_mutex_init(&tsBalanceMutex, NULL);
   balanceInitDnodeList();
@@ -667,8 +657,6 @@ int32_t balanceDropDnode(SDnodeObj *pDnode) {
     mnodeDecDnodeRef(pTempDnode);
   }
 
-  sdbFreeIter(pIter);
-
   if (pDnode->openVnodes > totalFreeVnodes) {
     mError("dnode:%d, openVnodes:%d totalFreeVnodes:%d no enough dnodes", pDnode->dnodeId, pDnode->openVnodes, totalFreeVnodes);
     return TSDB_CODE_MND_NO_ENOUGH_DNODES;
@@ -781,7 +769,11 @@ void balanceAccquireDnodeList() {
   int32_t    dnodeIndex = 0;
 
   while (1) {
-    if (dnodeIndex >= dnodesNum) break;
+    if (dnodeIndex >= dnodesNum) {
+      mnodeCancelGetNextDnode(pIter);
+      break;
+    }
+
     pIter = mnodeGetNextDnode(pIter, &pDnode);
     if (pDnode == NULL) break;
     if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
@@ -802,8 +794,6 @@ void balanceAccquireDnodeList() {
     dnodeIndex++;
   }
 
-  sdbFreeIter(pIter);
-
   tsBalanceDnodeListSize = dnodeIndex;
 }
 

src/client/inc/tsclient.h

@@ -282,7 +282,7 @@ typedef struct {
 
   int8_t       dataSourceType;     // load data from file or not
   int8_t       submitSchema; // submit block is built with table schema
-  STagData     tagData;
+  STagData    *pTagData;     // NOTE: pTagData->data is used as a variant length array
   SHashObj    *pTableList;   // referred table involved in sql
   SArray      *pDataBlocks;  // SArray<STableDataBlocks*> submit data blocks after parsing sql
 } SSqlCmd;

src/client/src/tscParseInsert.c

@@ -790,9 +790,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
     sql += index;
 
     tscAllocPayload(pCmd, sizeof(STagData));
-    STagData *pTag = &pCmd->tagData;
-
-    memset(pTag, 0, sizeof(STagData));
     
     //the source super table is moved to the secondary position of the pTableMetaInfo list
     if (pQueryInfo->numOfTables < 2) {
@@ -805,7 +802,14 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
       return code;
     }
 
+    STagData *pTag = realloc(pCmd->pTagData, offsetof(STagData, data));
+    if (pTag == NULL) {
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    }
+    memset(pTag, 0, offsetof(STagData, data));
     tstrncpy(pTag->name, pSTableMeterMetaInfo->name, sizeof(pTag->name));
+    pCmd->pTagData = pTag;
+
     code = tscGetTableMeta(pSql, pSTableMeterMetaInfo);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
@@ -934,7 +938,13 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
       return TSDB_CODE_TSC_OUT_OF_MEMORY;
     }
     tdSortKVRowByColIdx(row);
-    pTag->dataLen = kvRowLen(row);
+
+    pTag = (STagData*)realloc(pCmd->pTagData, offsetof(STagData, data) + kvRowLen(row));
+    if (pTag == NULL) {
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    }
+    pCmd->pTagData = pTag;
+    pTag->dataLen = htonl(kvRowLen(row));
     kvRowCpy(pTag->data, row);
     free(row);
 
@@ -945,8 +955,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) {
       return tscSQLSyntaxErrMsg(pCmd->payload, ") expected", sToken.z);
     }
 
-    pTag->dataLen = htonl(pTag->dataLen);
-
     if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) {
       return tscInvalidSQLErrMsg(pCmd->payload, "invalid table name", *sqlstr);
     }

src/client/src/tscServer.c

@@ -1565,11 +1565,11 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
   char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg);
 
-  size_t len = htonl(pCmd->tagData.dataLen);
-  if (pSql->cmd.autoCreated) {
+  if (pCmd->autoCreated && pCmd->pTagData != NULL) {
+    int len = htonl(pCmd->pTagData->dataLen);
     if (len > 0) {
-      len += sizeof(pCmd->tagData.name) + sizeof(pCmd->tagData.dataLen);
-      memcpy(pInfoMsg->tags, &pCmd->tagData, len);
+      len += sizeof(pCmd->pTagData->name) + sizeof(pCmd->pTagData->dataLen);
+      memcpy(pInfoMsg->tags, pCmd->pTagData, len);
       pMsg += len;
     }
   }
@@ -2239,8 +2239,6 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
   pNew->signature = pNew;
   pNew->cmd.command = TSDB_SQL_META;
 
-  registerSqlObj(pNew);
-
   tscAddSubqueryInfo(&pNew->cmd);
 
   SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0);
@@ -2248,8 +2246,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
   pNew->cmd.autoCreated = pSql->cmd.autoCreated;  // create table if not exists
   if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) {
     tscError("%p malloc failed for payload to get table meta", pSql);
-    free(pNew);
-
+    tscFreeSqlObj(pNew);
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 
@@ -2257,12 +2254,25 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
   assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1);
 
   tstrncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, sizeof(pNewMeterMetaInfo->name));
-  memcpy(&pNew->cmd.tagData, &pSql->cmd.tagData, sizeof(pSql->cmd.tagData));
+
+  if (pSql->cmd.pTagData != NULL) {
+    int size = offsetof(STagData, data) + htonl(pSql->cmd.pTagData->dataLen);
+    pNew->cmd.pTagData = calloc(1, size);
+    if (pNew->cmd.pTagData == NULL) {
+      tscError("%p malloc failed for new tag data to get table meta", pSql);
+      tscFreeSqlObj(pNew);
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    }
+    memcpy(pNew->cmd.pTagData, pSql->cmd.pTagData, size);
+  }
+
   tscDebug("%p new pSqlObj:%p to get tableMeta, auto create:%d", pSql, pNew, pNew->cmd.autoCreated);
 
   pNew->fp = tscTableMetaCallBack;
   pNew->param = pSql;
 
+  registerSqlObj(pNew);
+
   int32_t code = tscProcessSql(pNew);
   if (code == TSDB_CODE_SUCCESS) {
     code = TSDB_CODE_TSC_ACTION_IN_PROGRESS;  // notify upper application that current process need to be terminated

src/client/src/tscSub.c

@@ -442,7 +442,12 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) {
 
   size_t size = taosArrayGetSize(pSub->progress) * sizeof(STableIdInfo);
   size += sizeof(SQueryTableMsg) + 4096;
-  tscAllocPayload(&pSql->cmd, (int)size);
+  int code = tscAllocPayload(&pSql->cmd, (int)size);
+  if (code != TSDB_CODE_SUCCESS) {
+    tscError("failed to alloc payload");
+    return NULL;
+  }
+
   for (int retry = 0; retry < 3; retry++) {
     tscRemoveFromSqlList(pSql);
 

src/client/src/tscUtil.c

@@ -512,6 +512,8 @@ void tscFreeSqlObj(SSqlObj* pSql) {
   tscFreeSqlResult(pSql);
   tscResetSqlCmdObj(pCmd, false);
 
+  tfree(pCmd->pTagData);
+  
   memset(pCmd->payload, 0, (size_t)pCmd->allocSize);
   tfree(pCmd->payload);
   pCmd->allocSize = 0;
@@ -1909,7 +1911,17 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm
   pCmd->command = cmd;
   pCmd->parseFinished = 1;
   pCmd->autoCreated = pSql->cmd.autoCreated;
-  memcpy(&pCmd->tagData, &pSql->cmd.tagData, sizeof(pCmd->tagData));
+
+  if (pSql->cmd.pTagData != NULL) {
+    int size = offsetof(STagData, data) + htonl(pSql->cmd.pTagData->dataLen);
+    pNew->cmd.pTagData = calloc(1, size);
+    if (pNew->cmd.pTagData == NULL) {
+      tscError("%p new subquery failed, unable to malloc tag data, tableIndex:%d", pSql, 0);
+      free(pNew);
+      return NULL;
+    }
+    memcpy(pNew->cmd.pTagData, pSql->cmd.pTagData, size);
+  }
 
   if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) {
     tscFreeSqlObj(pNew);

src/common/inc/tglobal.h

@@ -97,6 +97,7 @@ extern int32_t tsAlternativeRole;
 extern int32_t tsBalanceInterval;
 extern int32_t tsOfflineThreshold;
 extern int32_t tsMnodeEqualVnodeNum;
+extern int32_t tsFlowCtrl;
 
 // restful
 extern int32_t  tsEnableHttpModule;

src/common/src/tglobal.c

@@ -133,6 +133,7 @@ int32_t tsAlternativeRole = 0;
 int32_t tsBalanceInterval = 300;  // seconds
 int32_t tsOfflineThreshold = 86400*100;   // seconds 10days
 int32_t tsMnodeEqualVnodeNum = 4;
+int32_t tsFlowCtrl = 1;
 
 // restful
 int32_t  tsEnableHttpModule = 1;
@@ -199,7 +200,7 @@ int32_t tsNumOfLogLines = 10000000;
 int32_t mDebugFlag = 135;
 int32_t sdbDebugFlag = 135;
 int32_t dDebugFlag = 135;
-int32_t vDebugFlag = 131;
+int32_t vDebugFlag = 135;
 int32_t cDebugFlag = 131;
 int32_t jniDebugFlag = 131;
 int32_t odbcDebugFlag = 131;
@@ -971,6 +972,17 @@ static void doInitGlobalConfig(void) {
   cfg.maxValue = 1000;
   cfg.ptrLength = 0;
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
+    // module configs
+  cfg.option = "flowctrl";
+  cfg.ptr = &tsFlowCtrl;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = 0;
+  cfg.maxValue = 1;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
   cfg.option = "http";

go @ 8c58c512

-Subproject commit 050667e5b4d0eafa5387e4283e713559b421203f
+Subproject commit 8c58c512b6acda8bcdfa48fdc7140227b5221766

grafanaplugin @ d598db16

-Subproject commit ec77d9049a719dabfd1a7c1122a209e201861944
+Subproject commit d598db167eb256fe67409b7bb3d0eb7fffc3ff8c

src/connector/jdbc/CMakeLists.txt

@@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED)
   ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME}
     POST_BUILD
     COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
-    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.8-dist.jar ${LIBRARY_OUTPUT_PATH}
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.13-dist.jar ${LIBRARY_OUTPUT_PATH}
     COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
     COMMENT "build jdbc driver")
   ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME})

src/connector/jdbc/deploy-pom.xml

@@ -5,7 +5,7 @@
 
   <groupId>com.taosdata.jdbc</groupId>
   <artifactId>taos-jdbcdriver</artifactId>
-  <version>2.0.10</version>
+  <version>2.0.13</version>
   <packaging>jar</packaging>
 
   <name>JDBCDriver</name>
@@ -49,17 +49,29 @@
         </exclusion>
       </exclusions>
     </dependency>
-    <dependency>
-      <groupId>org.apache.commons</groupId>
-      <artifactId>commons-lang3</artifactId>
-      <version>3.5</version>
-    </dependency>
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
       <version>4.13</version>
       <scope>test</scope>
     </dependency>
+
+    <!-- for restful -->
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpclient</artifactId>
+      <version>4.5.8</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+      <version>3.9</version>
+    </dependency>
+    <dependency>
+      <groupId>com.alibaba</groupId>
+      <artifactId>fastjson</artifactId>
+      <version>1.2.58</version>
+    </dependency>
   </dependencies>
 
   <build>

src/connector/jdbc/pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.taosdata.jdbc</groupId>
     <artifactId>taos-jdbcdriver</artifactId>
-    <version>2.0.8</version>
+    <version>2.0.13</version>
     <packaging>jar</packaging>
     <name>JDBCDriver</name>
     <url>https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc</url>
@@ -112,6 +112,13 @@
                 <artifactId>maven-surefire-plugin</artifactId>
                 <version>2.12.4</version>
                 <configuration>
+                    <includes>
+                        <include>**/*Test.java</include>
+                    </includes>
+                    <excludes>
+                        <exclude>**/BatchInsertTest.java</exclude>
+                        <exclude>**/FailOverTest.java</exclude>
+                    </excludes>
                     <testFailureIgnore>true</testFailureIgnore>
                 </configuration>
             </plugin>

src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java

@@ -15,7 +15,6 @@ public class RestfulConnection implements Connection {
     private final String database;
     private final String url;
 
-
     public RestfulConnection(String host, String port, Properties props, String database, String url) {
         this.host = host;
         this.port = Integer.parseInt(port);
@@ -28,7 +27,7 @@ public class RestfulConnection implements Connection {
     public Statement createStatement() throws SQLException {
         if (isClosed())
             throw new SQLException(TSDBConstants.WrapErrMsg("restful TDengine connection is closed."));
-        return new RestfulStatement(this, this.database);
+        return new RestfulStatement(this, database);
     }
 
     @Override
@@ -104,22 +103,28 @@ public class RestfulConnection implements Connection {
 
     @Override
     public void setTransactionIsolation(int level) throws SQLException {
-
+        //transaction is not supported
+        throw new SQLFeatureNotSupportedException("transactions are not supported");
     }
 
+    /**
+     *
+     */
     @Override
     public int getTransactionIsolation() throws SQLException {
-        return 0;
+        //Connection.TRANSACTION_NONE specifies that transactions are not supported.
+        return Connection.TRANSACTION_NONE;
     }
 
     @Override
     public SQLWarning getWarnings() throws SQLException {
+        //TODO: getWarnings not implemented
         return null;
     }
 
     @Override
     public void clearWarnings() throws SQLException {
-
+        throw new SQLFeatureNotSupportedException("clearWarnings not supported.");
     }
 
     @Override
@@ -209,22 +214,26 @@ public class RestfulConnection implements Connection {
 
     @Override
     public Clob createClob() throws SQLException {
-        return null;
+        //TODO: not supported
+        throw new SQLFeatureNotSupportedException();
     }
 
     @Override
     public Blob createBlob() throws SQLException {
-        return null;
+        //TODO: not supported
+        throw new SQLFeatureNotSupportedException();
     }
 
     @Override
     public NClob createNClob() throws SQLException {
-        return null;
+        //TODO: not supported
+        throw new SQLFeatureNotSupportedException();
     }
 
     @Override
     public SQLXML createSQLXML() throws SQLException {
-        return null;
+        //TODO: not supported
+        throw new SQLFeatureNotSupportedException();
     }
 
     @Override
@@ -254,12 +263,14 @@ public class RestfulConnection implements Connection {
 
     @Override
     public Array createArrayOf(String typeName, Object[] elements) throws SQLException {
-        return null;
+        //TODO: not supported
+        throw new SQLFeatureNotSupportedException();
     }
 
     @Override
     public Struct createStruct(String typeName, Object[] attributes) throws SQLException {
-        return null;
+        //TODO: not supported
+        throw new SQLFeatureNotSupportedException();
     }
 
     @Override
@@ -289,12 +300,16 @@ public class RestfulConnection implements Connection {
 
     @Override
     public <T> T unwrap(Class<T> iface) throws SQLException {
-        return null;
+        try {
+            return iface.cast(this);
+        } catch (ClassCastException cce) {
+            throw new SQLException("Unable to unwrap to " + iface.toString());
+        }
     }
 
     @Override
     public boolean isWrapperFor(Class<?> iface) throws SQLException {
-        return false;
+        return iface.isInstance(this);
     }
 
     public String getHost() {

src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java

@@ -35,7 +35,7 @@ public class RestfulDriver extends AbstractTaosDriver {
         Properties props = parseURL(url, info);
         String host = props.getProperty(TSDBDriver.PROPERTY_KEY_HOST, "localhost");
         String port = props.getProperty(TSDBDriver.PROPERTY_KEY_PORT, "6041");
-        String database = props.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME);
+        String database = props.containsKey(TSDBDriver.PROPERTY_KEY_DBNAME) ? props.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME) : null;
 
         String loginUrl = "http://" + props.getProperty(TSDBDriver.PROPERTY_KEY_HOST) + ":"
                 + props.getProperty(TSDBDriver.PROPERTY_KEY_PORT) + "/rest/login/"
@@ -86,6 +86,7 @@ public class RestfulDriver extends AbstractTaosDriver {
 
     @Override
     public Logger getParentLogger() throws SQLFeatureNotSupportedException {
-        return null;
+        //TODO SQLFeatureNotSupportedException
+        throw new SQLFeatureNotSupportedException();
     }
 }

src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java

@@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
 import com.taosdata.jdbc.TSDBConstants;
 import com.taosdata.jdbc.rs.util.HttpClientPoolUtil;
+import com.taosdata.jdbc.utils.SqlSyntaxValidator;
 
 import java.sql.*;
 import java.util.Arrays;
@@ -11,19 +12,23 @@ import java.util.List;
 
 public class RestfulStatement implements Statement {
 
-    private final String catalog;
+    private boolean closed;
+    private String database;
     private final RestfulConnection conn;
 
-    public RestfulStatement(RestfulConnection c, String catalog) {
+    public RestfulStatement(RestfulConnection c, String database) {
         this.conn = c;
-        this.catalog = catalog;
+        this.database = database;
     }
 
     @Override
     public ResultSet executeQuery(String sql) throws SQLException {
+        if (isClosed())
+            throw new SQLException("statement already closed");
+        if (!SqlSyntaxValidator.isSelectSql(sql))
+            throw new SQLException("not a select sql for executeQuery: " + sql);
 
-        final String url = "http://" + conn.getHost() + ":"+conn.getPort()+"/rest/sql";
-
+        final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql";
         String result = HttpClientPoolUtil.execute(url, sql);
         String fields = "";
         List<String> words = Arrays.asList(sql.split(" "));
@@ -65,12 +70,29 @@ public class RestfulStatement implements Statement {
 
     @Override
     public int executeUpdate(String sql) throws SQLException {
-        return 0;
+        if (isClosed())
+            throw new SQLException("statement already closed");
+        if (!SqlSyntaxValidator.isValidForExecuteUpdate(sql))
+            throw new SQLException("not a valid sql for executeUpdate: " + sql);
+
+        if (this.database == null)
+            throw new SQLException("Database not specified or available");
+
+        final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql";
+        HttpClientPoolUtil.execute(url, "use " + conn.getDatabase());
+        String result = HttpClientPoolUtil.execute(url, sql);
+        JSONObject jsonObject = JSON.parseObject(result);
+        if (jsonObject.getString("status").equals("error")) {
+            throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " +
+                    jsonObject.getString("desc") + "\n" +
+                    "error code: " + jsonObject.getString("code")));
+        }
+        return Integer.parseInt(jsonObject.getString("rows"));
     }
 
     @Override
     public void close() throws SQLException {
-
+        this.closed = true;
     }
 
     @Override
@@ -115,6 +137,7 @@ public class RestfulStatement implements Statement {
 
     @Override
     public SQLWarning getWarnings() throws SQLException {
+        //TODO: getWarnings not Implemented
         return null;
     }
 
@@ -130,7 +153,29 @@ public class RestfulStatement implements Statement {
 
     @Override
     public boolean execute(String sql) throws SQLException {
-        return false;
+        if (isClosed()) {
+            throw new SQLException("Invalid method call on a closed statement.");
+        }
+        //如果执行了use操作应该将当前Statement的catalog设置为新的database
+        if (SqlSyntaxValidator.isUseSql(sql)) {
+            this.database = sql.trim().replace("use", "").trim();
+        }
+        if (this.database == null)
+            throw new SQLException("Database not specified or available");
+
+        final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql";
+        // use database
+        HttpClientPoolUtil.execute(url, "use " + conn.getDatabase());
+        // execute sql
+        String result = HttpClientPoolUtil.execute(url, sql);
+        // parse result
+        JSONObject jsonObject = JSON.parseObject(result);
+        if (jsonObject.getString("status").equals("error")) {
+            throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " +
+                    jsonObject.getString("desc") + "\n" +
+                    "error code: " + jsonObject.getString("code")));
+        }
+        return true;
     }
 
     @Override
@@ -245,7 +290,7 @@ public class RestfulStatement implements Statement {
 
     @Override
     public boolean isClosed() throws SQLException {
-        return false;
+        return closed;
     }
 
     @Override
@@ -270,11 +315,15 @@ public class RestfulStatement implements Statement {
 
     @Override
     public <T> T unwrap(Class<T> iface) throws SQLException {
-        return null;
+        try {
+            return iface.cast(this);
+        } catch (ClassCastException cce) {
+            throw new SQLException("Unable to unwrap to " + iface.toString());
+        }
     }
 
     @Override
     public boolean isWrapperFor(Class<?> iface) throws SQLException {
-        return false;
+        return iface.isInstance(this);
     }
 }

src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java

@@ -22,6 +22,9 @@ import java.sql.SQLException;
 
 public class SqlSyntaxValidator {
 
+    private static final String[] updateSQL = {"insert", "update", "delete", "create", "alter", "drop", "show", "describe", "use"};
+    private static final String[] querySQL = {"select"};
+
     private TSDBConnection tsdbConnection;
 
     public SqlSyntaxValidator(Connection connection) {
@@ -43,4 +46,28 @@ public class SqlSyntaxValidator {
         }
         return res;
     }
+
+    public static boolean isValidForExecuteUpdate(String sql) {
+        for (String prefix : updateSQL) {
+            if (sql.trim().toLowerCase().startsWith(prefix))
+                return true;
+        }
+        return false;
+    }
+
+    public static boolean isUseSql(String sql) {
+        return sql.trim().toLowerCase().startsWith(updateSQL[8]) || sql.trim().toLowerCase().matches("create\\s*database.*") || sql.toLowerCase().toLowerCase().matches("drop\\s*database.*");
+    }
+
+    public static boolean isUpdateSql(String sql) {
+        return sql.trim().toLowerCase().startsWith(updateSQL[1]);
+    }
+
+    public static boolean isInsertSql(String sql) {
+        return sql.trim().toLowerCase().startsWith(updateSQL[0]);
+    }
+
+    public static boolean isSelectSql(String sql) {
+        return sql.trim().toLowerCase().startsWith(querySQL[0]);
+    }
 }

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java

+package com.taosdata.jdbc.cases;
+
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+public class AppMemoryLeakTest {
+
+    @Test(expected = SQLException.class)
+    public void testCreateTooManyConnection() throws ClassNotFoundException, SQLException {
+        Class.forName("com.taosdata.jdbc.TSDBDriver");
+        int conCnt = 0;
+        while (true) {
+            Connection conn = DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata");
+            System.out.println(conCnt++ + " : " + conn);
+        }
+    }
+
+    @Test
+    public void testCreateTooManyStatement() throws ClassNotFoundException, SQLException {
+        Class.forName("com.taosdata.jdbc.TSDBDriver");
+        int stmtCnt = 0;
+        Connection conn = DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata");
+        while (true) {
+            Statement stmt = conn.createStatement();
+            System.out.println(++stmtCnt + " : " + stmt);
+        }
+    }
+
+    public static void main(String[] args) throws ClassNotFoundException, SQLException {
+        Class.forName("com.taosdata.jdbc.TSDBDriver");
+        int stmtCnt = 0;
+        Connection conn = DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata");
+        while (true) {
+            Statement stmt = conn.createStatement();
+            System.out.println(++stmtCnt + " : " + stmt);
+        }
+    }
+
+
+}

src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java

@@ -8,33 +8,43 @@ import java.sql.*;
 public class RestfulDriverTest {
 
     @Test
-    public void testCase001() {
-        try {
-            Class.forName("com.taosdata.jdbc.rs.RestfulDriver");
-            Connection connection = DriverManager.getConnection("jdbc:TAOS-RS://master:6041/?user=root&password=taosdata");
-            Statement statement = connection.createStatement();
-            ResultSet resultSet = statement.executeQuery("select * from log.log");
-            ResultSetMetaData metaData = resultSet.getMetaData();
-            while (resultSet.next()) {
-                for (int i = 1; i <= metaData.getColumnCount(); i++) {
-                    String column = metaData.getColumnLabel(i);
-                    String value = resultSet.getString(i);
-                    System.out.print(column + ":" + value + "\t");
-                }
-                System.out.println();
-            }
-            statement.close();
-            connection.close();
-        } catch (SQLException | ClassNotFoundException e) {
-            e.printStackTrace();
-        }
+    public void connect() {
+
     }
 
     @Test
-    public void testAcceptUrl() throws SQLException {
+    public void acceptsURL() throws SQLException {
         Driver driver = new RestfulDriver();
         boolean isAccept = driver.acceptsURL("jdbc:TAOS-RS://master:6041");
         Assert.assertTrue(isAccept);
+        isAccept = driver.acceptsURL("jdbc:TAOS://master:6041");
+        Assert.assertFalse(isAccept);
+    }
+
+    @Test
+    public void getPropertyInfo() throws SQLException {
+        Driver driver = new RestfulDriver();
+        final String url = "";
+        DriverPropertyInfo[] propertyInfo = driver.getPropertyInfo(url, null);
+    }
+
+    @Test
+    public void getMajorVersion() {
+        Assert.assertEquals(2, new RestfulDriver().getMajorVersion());
+    }
+
+    @Test
+    public void getMinorVersion() {
+        Assert.assertEquals(0, new RestfulDriver().getMinorVersion());
     }
 
+    @Test
+    public void jdbcCompliant() {
+        Assert.assertFalse(new RestfulDriver().jdbcCompliant());
+    }
+
+    @Test(expected = SQLFeatureNotSupportedException.class)
+    public void getParentLogger() throws SQLFeatureNotSupportedException {
+        new RestfulDriver().getParentLogger();
+    }
 }

src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java

+package com.taosdata.jdbc.rs;
+
+import org.junit.*;
+import org.junit.runners.MethodSorters;
+
+import java.sql.*;
+import java.util.Random;
+
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class RestfulJDBCTest {
+
+    private Connection connection;
+
+    @Before
+    public void before() throws ClassNotFoundException, SQLException {
+        Class.forName("com.taosdata.jdbc.rs.RestfulDriver");
+        connection = DriverManager.getConnection("jdbc:TAOS-RS://master:6041/restful_test?user=root&password=taosdata");
+    }
+
+    @After
+    public void after() throws SQLException {
+        if (connection != null)
+            connection.close();
+    }
+
+
+    /**
+     * 查询所有log.log
+     **/
+    @Test
+    public void testCase001() {
+        try {
+            Statement statement = connection.createStatement();
+            ResultSet resultSet = statement.executeQuery("select * from log.log");
+            ResultSetMetaData metaData = resultSet.getMetaData();
+            while (resultSet.next()) {
+                for (int i = 1; i <= metaData.getColumnCount(); i++) {
+                    String column = metaData.getColumnLabel(i);
+                    String value = resultSet.getString(i);
+                    System.out.print(column + ":" + value + "\t");
+                }
+                System.out.println();
+            }
+            statement.close();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * create database
+     */
+    @Test
+    public void testCase002() {
+        try (Statement stmt = connection.createStatement()) {
+            stmt.execute("drop database if exists restful_test");
+            stmt.execute("create database if not exists restful_test");
+            stmt.execute("use restful_test");
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * create super table
+     ***/
+    @Test
+    public void testCase003() {
+        try (Statement stmt = connection.createStatement()) {
+            stmt.execute("create table weather(ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int)");
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+    @Test
+    public void testCase004() {
+        try (Statement stmt = connection.createStatement()) {
+            for (int i = 1; i <= 100; i++) {
+                stmt.execute("create table t" + i + " using weather tags('beijing', '" + i + "')");
+            }
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+    private Random random = new Random(System.currentTimeMillis());
+
+    @Test
+    public void testCase005() {
+        try (Statement stmt = connection.createStatement()) {
+            int rows = 0;
+            for (int i = 0; i < 10; i++) {
+                for (int j = 1; j <= 100; j++) {
+                    long currentTimeMillis = System.currentTimeMillis();
+                    int affectRows = stmt.executeUpdate("insert into t" + j + " values(" + currentTimeMillis + "," + (random.nextFloat() * 50) + "," + random.nextInt(100) + ")");
+                    Assert.assertEquals(1, affectRows);
+                    rows += affectRows;
+                }
+            }
+            Assert.assertEquals(1000, rows);
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+
+}

src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java

+package com.taosdata.jdbc.utils;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class SqlSyntaxValidatorTest {
+
+    @Test
+    public void validateSqlSyntax() {
+    }
+
+    @Test
+    public void isSelectSQL() {
+        Assert.assertTrue(SqlSyntaxValidator.isSelectSql("select * from test.weather"));
+        Assert.assertTrue(SqlSyntaxValidator.isSelectSql(" select * from test.weather"));
+        Assert.assertTrue(SqlSyntaxValidator.isSelectSql(" select * from test.weather "));
+        Assert.assertFalse(SqlSyntaxValidator.isSelectSql("insert into test.weather values(now, 1.1, 2)"));
+    }
+
+    @Test
+    public void isUseSQL() {
+        Assert.assertTrue(SqlSyntaxValidator.isUseSql("use database test"));
+        Assert.assertTrue(SqlSyntaxValidator.isUseSql("create database test"));
+        Assert.assertTrue(SqlSyntaxValidator.isUseSql("create database if not exist test"));
+        Assert.assertTrue(SqlSyntaxValidator.isUseSql("drop database test"));
+        Assert.assertTrue(SqlSyntaxValidator.isUseSql("drop database if exist test"));
+    }
+}
\ No newline at end of file

src/connector/python/linux/python2/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.0",
+    version="2.0.2",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",

src/connector/python/linux/python2/taos/cinterface.py

@@ -18,7 +18,7 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False):
         _timestamp_converter = _convert_microsecond_to_datetime
 
     if num_of_rows > 0:
-        return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)][::1]))
+        return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]))
     else:
         return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]))
 
@@ -26,7 +26,7 @@ def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C bool row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ]
 
@@ -34,7 +34,7 @@ def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C tinyint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     
@@ -42,7 +42,7 @@ def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C smallint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)][::1]]
+        return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]]
     else:
         return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ]
 
@@ -50,7 +50,7 @@ def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C int row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ]
 
@@ -58,7 +58,7 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C bigint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ]
 
@@ -66,7 +66,7 @@ def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C float row to python row
     """
     if num_of_rows > 0:
-        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)][::1] ]
+        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ]
     else:
         return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ]
 
@@ -74,7 +74,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C double row to python row
     """
     if num_of_rows > 0:
-        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)][::1] ]
+        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ]
     else:
         return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ]
 
@@ -82,7 +82,7 @@ def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)][::1]]
+        return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
     else:
         return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
 
@@ -90,9 +90,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C nchar row to python row
     """
     assert(nbytes is not None)
-
-    res = []
-
+    res=[]
     for i in range(abs(num_of_rows)):
         try:
             if num_of_rows >= 0:
@@ -104,16 +102,48 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
             res.append(None)
 
     return res 
-    # if num_of_rows > 0:
-    #     for i in range(abs(num_of_rows)):
-    #         try:
-    #             res.append( (ctypes.cast(data+nbytes*i,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
-    #         except ValueError:
-    #             res.append(None)
-    #     return res
-    #         # return [ele.value for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)][::-1]]
-    # else:
-    #     return [ele.value for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)]]
+
+def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False):
+    """Function to convert C binary row to python row
+    """
+    res=[]
+    if num_of_rows > 0:
+        for i in range(abs(num_of_rows)):
+            try:
+                rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop()
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode()[0:rbyte] )
+            except ValueError:
+                res.append(None)
+    else:
+        for i in range(abs(num_of_rows)):
+            try:
+                rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop()
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode()[0:rbyte] )
+            except ValueError:
+                res.append(None)
+    return res
+
+def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False):
+    """Function to convert C nchar row to python row
+    """
+    assert(nbytes is not None)
+    res=[]
+    if num_of_rows >= 0:
+        for i in range(abs(num_of_rows)):
+            try:
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode() )
+            except ValueError:
+                res.append(None)
+    else:
+        for i in range(abs(num_of_rows)):
+            try:
+                res.append( (ctypes.cast(data+nbytes*i+2,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
+            except ValueError:
+                res.append(None)
+    return res
 
 _CONVERT_FUNC = {
     FieldType.C_BOOL: _crow_bool_to_python,
@@ -128,6 +158,19 @@ _CONVERT_FUNC = {
     FieldType.C_NCHAR : _crow_nchar_to_python
 }
 
+_CONVERT_FUNC_BLOCK = {
+    FieldType.C_BOOL: _crow_bool_to_python,
+    FieldType.C_TINYINT : _crow_tinyint_to_python,
+    FieldType.C_SMALLINT : _crow_smallint_to_python,
+    FieldType.C_INT : _crow_int_to_python,
+    FieldType.C_BIGINT : _crow_bigint_to_python,
+    FieldType.C_FLOAT : _crow_float_to_python,
+    FieldType.C_DOUBLE : _crow_double_to_python,
+    FieldType.C_BINARY: _crow_binary_to_python_block,
+    FieldType.C_TIMESTAMP : _crow_timestamp_to_python, 
+    FieldType.C_NCHAR : _crow_nchar_to_python_block
+}
+
 # Corresponding TAOS_FIELD structure in C
 class TaosField(ctypes.Structure):
     _fields_ = [('name', ctypes.c_char * 65),
@@ -227,8 +270,8 @@ class CTaosInterface(object):
             print('connect to TDengine failed')
             raise ConnectionError("connect to TDengine failed")
             # sys.exit(1)
-        else:
-            print('connect to TDengine success')
+        #else:
+        #    print('connect to TDengine success')
 
         return connection
 
@@ -237,7 +280,7 @@ class CTaosInterface(object):
         '''Close the TDengine handle
         '''
         CTaosInterface.libtaos.taos_close(connection)
-        print('connection is closed')
+        #print('connection is closed')
 
     @staticmethod
     def query(connection, sql):
@@ -310,6 +353,24 @@ class CTaosInterface(object):
 
     @staticmethod
     def fetchBlock(result, fields):
+        pblock = ctypes.c_void_p(0)
+        num_of_rows = CTaosInterface.libtaos.taos_fetch_block(
+            result, ctypes.byref(pblock))
+        if num_of_rows == 0:
+            return None, 0
+        isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO)
+        blocks = [None] * len(fields)
+        fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result)
+        fieldLen = [ele for ele in ctypes.cast(fieldL,  ctypes.POINTER(ctypes.c_int))[:len(fields)]]
+        for i in range(len(fields)):
+            data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i]
+            if fields[i]['type'] not in _CONVERT_FUNC_BLOCK:
+                raise DatabaseError("Invalid data type returned from database")
+            blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro)
+
+        return blocks, abs(num_of_rows)
+    @staticmethod
+    def fetchRow(result, fields):
         pblock = ctypes.c_void_p(0)
         pblock = CTaosInterface.libtaos.taos_fetch_row(result)  
         if pblock : 
@@ -393,7 +454,7 @@ class CTaosInterface(object):
     def errStr(result):
         """Return the error styring
         """
-        return CTaosInterface.libtaos.taos_errstr(result)
+        return CTaosInterface.libtaos.taos_errstr(result).decode('utf-8')
 
 
 if __name__ == '__main__':

src/connector/python/linux/python2/taos/cursor.py

@@ -49,7 +49,7 @@ class TDengineCursor(object):
             raise OperationalError("Invalid use of fetch iterator")
 
         if self._block_rows <= self._block_iter:
-            block, self._block_rows = CTaosInterface.fetchBlock(
+            block, self._block_rows = CTaosInterface.fetchRow(
                 self._result, self._fields)
             if self._block_rows == 0:
                 raise StopIteration
@@ -193,7 +193,7 @@ class TDengineCursor(object):
         buffer = [[] for i in range(len(self._fields))]
         self._rowcount = 0
         while True:
-            block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields)
+            block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields)
             errno = CTaosInterface.libtaos.taos_errno(self._result)
             if errno != 0:
                 raise ProgrammingError(CTaosInterface.errStr(self._result), errno)
@@ -203,7 +203,23 @@ class TDengineCursor(object):
             for i in range(len(self._fields)):
                 buffer[i].extend(block[i])
         return list(map(tuple, zip(*buffer)))
+    def fetchall_block(self):
+        if self._result is None or self._fields is None:
+            raise OperationalError("Invalid use of fetchall")
 
+        buffer = [[] for i in range(len(self._fields))]
+        self._rowcount = 0
+        while True:
+            block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields)
+            errno = CTaosInterface.libtaos.taos_errno(self._result)
+            if errno != 0:
+                raise ProgrammingError(CTaosInterface.errStr(self._result), errno)
+            if num_of_fields == 0:
+                break
+            self._rowcount += num_of_fields
+            for i in range(len(self._fields)):
+                buffer[i].extend(block[i])
+        return list(map(tuple, zip(*buffer)))
     def nextset(self):
         """
         """

src/connector/python/linux/python3/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.0",
+    version="2.0.2",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",

src/connector/python/linux/python3/taos/cinterface.py

@@ -18,7 +18,7 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False):
         _timestamp_converter = _convert_microsecond_to_datetime
 
     if num_of_rows > 0:
-        return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)][::1]))
+        return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]))
     else:
         return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)]))
 
@@ -26,7 +26,7 @@ def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C bool row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ]
 
@@ -34,7 +34,7 @@ def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C tinyint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     
@@ -42,7 +42,7 @@ def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C smallint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)][::1]]
+        return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]]
     else:
         return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ]
 
@@ -50,7 +50,7 @@ def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C int row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ]
 
@@ -58,7 +58,7 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C bigint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ]
 
@@ -66,7 +66,7 @@ def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C float row to python row
     """
     if num_of_rows > 0:
-        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)][::1] ]
+        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ]
     else:
         return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ]
 
@@ -74,7 +74,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C double row to python row
     """
     if num_of_rows > 0:
-        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)][::1] ]
+        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ]
     else:
         return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ]
 
@@ -82,7 +82,7 @@ def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)][::1]]
+        return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
     else:
         return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
 
@@ -90,9 +90,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C nchar row to python row
     """
     assert(nbytes is not None)
-
-    res = []
-
+    res=[]
     for i in range(abs(num_of_rows)):
         try:
             if num_of_rows >= 0:
@@ -104,16 +102,48 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
             res.append(None)
 
     return res 
-    # if num_of_rows > 0:
-    #     for i in range(abs(num_of_rows)):
-    #         try:
-    #             res.append( (ctypes.cast(data+nbytes*i,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
-    #         except ValueError:
-    #             res.append(None)
-    #     return res
-    #         # return [ele.value for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)][::1]]
-    # else:
-    #     return [ele.value for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)]]
+
+def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False):
+    """Function to convert C binary row to python row
+    """
+    res=[]
+    if num_of_rows > 0:
+        for i in range(abs(num_of_rows)):
+            try:
+                rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop()
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode()[0:rbyte] )
+            except ValueError:
+                res.append(None)
+    else:
+        for i in range(abs(num_of_rows)):
+            try:
+                rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop()
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode()[0:rbyte] )
+            except ValueError:
+                res.append(None)
+    return res
+
+def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False):
+    """Function to convert C nchar row to python row
+    """
+    assert(nbytes is not None)
+    res=[]
+    if num_of_rows >= 0:
+        for i in range(abs(num_of_rows)):
+            try:
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode() )
+            except ValueError:
+                res.append(None)
+    else:
+        for i in range(abs(num_of_rows)):
+            try:
+                res.append( (ctypes.cast(data+nbytes*i+2,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
+            except ValueError:
+                res.append(None)
+    return res
 
 _CONVERT_FUNC = {
     FieldType.C_BOOL: _crow_bool_to_python,
@@ -128,6 +158,19 @@ _CONVERT_FUNC = {
     FieldType.C_NCHAR : _crow_nchar_to_python
 }
 
+_CONVERT_FUNC_BLOCK = {
+    FieldType.C_BOOL: _crow_bool_to_python,
+    FieldType.C_TINYINT : _crow_tinyint_to_python,
+    FieldType.C_SMALLINT : _crow_smallint_to_python,
+    FieldType.C_INT : _crow_int_to_python,
+    FieldType.C_BIGINT : _crow_bigint_to_python,
+    FieldType.C_FLOAT : _crow_float_to_python,
+    FieldType.C_DOUBLE : _crow_double_to_python,
+    FieldType.C_BINARY: _crow_binary_to_python_block,
+    FieldType.C_TIMESTAMP : _crow_timestamp_to_python, 
+    FieldType.C_NCHAR : _crow_nchar_to_python_block
+}
+
 # Corresponding TAOS_FIELD structure in C
 class TaosField(ctypes.Structure):
     _fields_ = [('name', ctypes.c_char * 65),
@@ -308,29 +351,26 @@ class CTaosInterface(object):
 
         return fields
 
-    # @staticmethod
-    # def fetchBlock(result, fields):
-    #     pblock = ctypes.c_void_p(0)
-    #     num_of_rows = CTaosInterface.libtaos.taos_fetch_block(
-    #         result, ctypes.byref(pblock))
-    #     if num_of_rows == 0:
-    #         return None, 0
-
-    #     isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO)
-    #     blocks = [None] * len(fields)
-    #     fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result)
-    #     fieldLen = [ele for ele in ctypes.cast(fieldL,  ctypes.POINTER(ctypes.c_int))[:len(fields)]]
-    #     for i in range(len(fields)):
-    #         data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i]
-
-    #         if fields[i]['type'] not in _CONVERT_FUNC:
-    #             raise DatabaseError("Invalid data type returned from database")
-    #         print('====================',fieldLen[i])
-    #         blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro)
-
-    #     return blocks, abs(num_of_rows)
     @staticmethod
     def fetchBlock(result, fields):
+        pblock = ctypes.c_void_p(0)
+        num_of_rows = CTaosInterface.libtaos.taos_fetch_block(
+            result, ctypes.byref(pblock))
+        if num_of_rows == 0:
+            return None, 0
+        isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO)
+        blocks = [None] * len(fields)
+        fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result)
+        fieldLen = [ele for ele in ctypes.cast(fieldL,  ctypes.POINTER(ctypes.c_int))[:len(fields)]]
+        for i in range(len(fields)):
+            data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i]
+            if fields[i]['type'] not in _CONVERT_FUNC_BLOCK:
+                raise DatabaseError("Invalid data type returned from database")
+            blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro)
+
+        return blocks, abs(num_of_rows)
+    @staticmethod
+    def fetchRow(result, fields):
         pblock = ctypes.c_void_p(0)
         pblock = CTaosInterface.libtaos.taos_fetch_row(result)  
         if pblock : 
@@ -350,6 +390,7 @@ class CTaosInterface(object):
         else:
             return None, 0
         return blocks, abs(num_of_rows)
+
     @staticmethod
     def freeResult(result):
         CTaosInterface.libtaos.taos_free_result(result)

src/connector/python/linux/python3/taos/cursor.py

@@ -5,7 +5,6 @@ import threading
 
 # querySeqNum = 0
 
-
 class TDengineCursor(object):
     """Database cursor which is used to manage the context of a fetch operation.
 
@@ -51,7 +50,7 @@ class TDengineCursor(object):
             raise OperationalError("Invalid use of fetch iterator")
 
         if self._block_rows <= self._block_iter:
-            block, self._block_rows = CTaosInterface.fetchBlock(
+            block, self._block_rows = CTaosInterface.fetchRow(
                 self._result, self._fields)
             if self._block_rows == 0:
                 raise StopIteration
@@ -196,18 +195,13 @@ class TDengineCursor(object):
     def fetchall(self):
         """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation.
         """
-        # if threading.get_ident() != self._threadId:
-        #     info ="[WARNING] Cursor fetchall:Thread ID not match,creater:"+str(self._threadId)+" caller:"+str(threading.get_ident())
-        #     raise OperationalError(info)
-            # print(info)
-            # return None
         if self._result is None or self._fields is None:
             raise OperationalError("Invalid use of fetchall")
 
         buffer = [[] for i in range(len(self._fields))]
         self._rowcount = 0
         while True:
-            block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields)
+            block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields)
             errno = CTaosInterface.libtaos.taos_errno(self._result)
             if errno != 0:
                 raise ProgrammingError(CTaosInterface.errStr(self._result), errno)
@@ -218,6 +212,22 @@ class TDengineCursor(object):
                 buffer[i].extend(block[i])
         return list(map(tuple, zip(*buffer)))
 
+    def fetchall_block(self):
+        if self._result is None or self._fields is None:
+            raise OperationalError("Invalid use of fetchall")
+
+        buffer = [[] for i in range(len(self._fields))]
+        self._rowcount = 0
+        while True:
+            block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields)
+            errno = CTaosInterface.libtaos.taos_errno(self._result)
+            if errno != 0:
+                raise ProgrammingError(CTaosInterface.errStr(self._result), errno)
+            if num_of_fields == 0: break
+            self._rowcount += num_of_fields
+            for i in range(len(self._fields)):
+                buffer[i].extend(block[i])
+        return list(map(tuple, zip(*buffer)))
     def nextset(self):
         """
         """

src/connector/python/windows/python2/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.0",
+    version="2.0.2",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",

src/connector/python/windows/python2/taos/cinterface.py

@@ -18,7 +18,7 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False):
         _timestamp_converter = _convert_microsecond_to_datetime
 
     if num_of_rows > 0:
-        return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)][::1]))
+        return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)]))
     else:
         return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)]))
 
@@ -26,7 +26,7 @@ def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C bool row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ]
 
@@ -34,7 +34,7 @@ def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C tinyint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     
@@ -42,7 +42,7 @@ def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C smallint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)][::1]]
+        return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]]
     else:
         return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ]
 
@@ -50,7 +50,7 @@ def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C int row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ]
 
@@ -58,7 +58,7 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C bigint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ]
 
@@ -66,7 +66,7 @@ def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C float row to python row
     """
     if num_of_rows > 0:
-        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)][::1] ]
+        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ]
     else:
         return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ]
 
@@ -74,7 +74,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C double row to python row
     """
     if num_of_rows > 0:
-        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)][::1] ]
+        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ]
     else:
         return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ]
 
@@ -82,7 +82,7 @@ def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)][::1]]
+        return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
     else:
         return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
 
@@ -90,9 +90,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C nchar row to python row
     """
     assert(nbytes is not None)
-
-    res = []
-
+    res=[]
     for i in range(abs(num_of_rows)):
         try:
             if num_of_rows >= 0:
@@ -104,16 +102,48 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
             res.append(None)
 
     return res 
-    # if num_of_rows > 0:
-    #     for i in range(abs(num_of_rows)):
-    #         try:
-    #             res.append( (ctypes.cast(data+nbytes*i,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
-    #         except ValueError:
-    #             res.append(None)
-    #     return res
-    #         # return [ele.value for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)][::-1]]
-    # else:
-    #     return [ele.value for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)]]
+
+def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False):
+    """Function to convert C binary row to python row
+    """
+    res=[]
+    if num_of_rows > 0:
+        for i in range(abs(num_of_rows)):
+            try:
+                rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop()
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode()[0:rbyte] )
+            except ValueError:
+                res.append(None)
+    else:
+        for i in range(abs(num_of_rows)):
+            try:
+                rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop()
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode()[0:rbyte] )
+            except ValueError:
+                res.append(None)
+    return res
+
+def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False):
+    """Function to convert C nchar row to python row
+    """
+    assert(nbytes is not None)
+    res=[]
+    if num_of_rows >= 0:
+        for i in range(abs(num_of_rows)):
+            try:
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode() )
+            except ValueError:
+                res.append(None)
+    else:
+        for i in range(abs(num_of_rows)):
+            try:
+                res.append( (ctypes.cast(data+nbytes*i+2,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
+            except ValueError:
+                res.append(None)
+    return res
 
 _CONVERT_FUNC = {
     FieldType.C_BOOL: _crow_bool_to_python,
@@ -128,6 +158,19 @@ _CONVERT_FUNC = {
     FieldType.C_NCHAR : _crow_nchar_to_python
 }
 
+_CONVERT_FUNC_BLOCK = {
+    FieldType.C_BOOL: _crow_bool_to_python,
+    FieldType.C_TINYINT : _crow_tinyint_to_python,
+    FieldType.C_SMALLINT : _crow_smallint_to_python,
+    FieldType.C_INT : _crow_int_to_python,
+    FieldType.C_BIGINT : _crow_bigint_to_python,
+    FieldType.C_FLOAT : _crow_float_to_python,
+    FieldType.C_DOUBLE : _crow_double_to_python,
+    FieldType.C_BINARY: _crow_binary_to_python_block,
+    FieldType.C_TIMESTAMP : _crow_timestamp_to_python, 
+    FieldType.C_NCHAR : _crow_nchar_to_python_block
+}
+
 # Corresponding TAOS_FIELD structure in C
 class TaosField(ctypes.Structure):
     _fields_ = [('name', ctypes.c_char * 65),
@@ -225,9 +268,10 @@ class CTaosInterface(object):
 
         if connection.value == None:
             print('connect to TDengine failed')
+            raise ConnectionError("connect to TDengine failed")
             # sys.exit(1)
-        else:
-            print('connect to TDengine success')
+        #else:
+        #    print('connect to TDengine success')
 
         return connection
 
@@ -236,7 +280,7 @@ class CTaosInterface(object):
         '''Close the TDengine handle
         '''
         CTaosInterface.libtaos.taos_close(connection)
-        print('connection is closed')
+        #print('connection is closed')
 
     @staticmethod
     def query(connection, sql):
@@ -309,6 +353,24 @@ class CTaosInterface(object):
 
     @staticmethod
     def fetchBlock(result, fields):
+        pblock = ctypes.c_void_p(0)
+        num_of_rows = CTaosInterface.libtaos.taos_fetch_block(
+            result, ctypes.byref(pblock))
+        if num_of_rows == 0:
+            return None, 0
+        isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO)
+        blocks = [None] * len(fields)
+        fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result)
+        fieldLen = [ele for ele in ctypes.cast(fieldL,  ctypes.POINTER(ctypes.c_int))[:len(fields)]]
+        for i in range(len(fields)):
+            data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i]
+            if fields[i]['type'] not in _CONVERT_FUNC_BLOCK:
+                raise DatabaseError("Invalid data type returned from database")
+            blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro)
+
+        return blocks, abs(num_of_rows)
+    @staticmethod
+    def fetchRow(result, fields):
         pblock = ctypes.c_void_p(0)
         pblock = CTaosInterface.libtaos.taos_fetch_row(result)  
         if pblock : 
@@ -393,7 +455,7 @@ class CTaosInterface(object):
     def errStr(result):
         """Return the error styring
         """
-        return CTaosInterface.libtaos.taos_errstr(result)
+        return CTaosInterface.libtaos.taos_errstr(result).decode('utf-8')
 
 
 if __name__ == '__main__':

src/connector/python/windows/python2/taos/cursor.py

@@ -50,7 +50,7 @@ class TDengineCursor(object):
             raise OperationalError("Invalid use of fetch iterator")
 
         if self._block_rows <= self._block_iter:
-            block, self._block_rows = CTaosInterface.fetchBlock(self._result, self._fields)
+            block, self._block_rows = CTaosInterface.fetchRow(self._result, self._fields)
             if self._block_rows == 0:
                 raise StopIteration
             self._block = list(map(tuple, zip(*block)))
@@ -144,6 +144,24 @@ class TDengineCursor(object):
         if self._result is None or self._fields is None:
             raise OperationalError("Invalid use of fetchall")
 
+        buffer = [[] for i in range(len(self._fields))]
+        self._rowcount = 0
+        while True:
+            block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields)
+            errno = CTaosInterface.libtaos.taos_errno(self._result)
+            if errno != 0:
+                raise ProgrammingError(CTaosInterface.errStr(self._result), errno)
+            if num_of_fields == 0:
+                break
+            self._rowcount += num_of_fields
+            for i in range(len(self._fields)):
+                buffer[i].extend(block[i])
+        return list(map(tuple, zip(*buffer)))
+        
+    def fetchall_block(self):
+        if self._result is None or self._fields is None:
+            raise OperationalError("Invalid use of fetchall")
+
         buffer = [[] for i in range(len(self._fields))]
         self._rowcount = 0
         while True:

src/connector/python/windows/python3/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="taos",
-    version="2.0.0",
+    version="2.0.2",
     author="Taosdata Inc.",
     author_email="support@taosdata.com",
     description="TDengine python client package",

src/connector/python/windows/python3/taos/cinterface.py

@@ -18,7 +18,7 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False):
         _timestamp_converter = _convert_microsecond_to_datetime
 
     if num_of_rows > 0:
-        return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)][::1]))
+        return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)]))
     else:
         return list(map(_timestamp_converter, ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)]))
 
@@ -26,7 +26,7 @@ def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C bool row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ]
 
@@ -34,7 +34,7 @@ def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C tinyint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ]
     
@@ -42,7 +42,7 @@ def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C smallint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)][::1]]
+        return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]]
     else:
         return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ]
 
@@ -50,7 +50,7 @@ def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C int row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ]
 
@@ -58,7 +58,7 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C bigint row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)][::1] ]
+        return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ]
     else:
         return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ]
 
@@ -66,7 +66,7 @@ def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C float row to python row
     """
     if num_of_rows > 0:
-        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)][::1] ]
+        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ]
     else:
         return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ]
 
@@ -74,7 +74,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C double row to python row
     """
     if num_of_rows > 0:
-        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)][::1] ]
+        return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ]
     else:
         return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data,  ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ]
 
@@ -82,7 +82,7 @@ def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False):
     """Function to convert C binary row to python row
     """
     if num_of_rows > 0:
-        return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)][::1]]
+        return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
     else:
         return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]]
 
@@ -104,16 +104,48 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False):
             res.append(None)
 
     return res
-    # if num_of_rows > 0:
-    #     for i in range(abs(num_of_rows)):
-    #         try:
-    #             res.append( (ctypes.cast(data+nbytes*i,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
-    #         except ValueError:
-    #             res.append(None)
-    #     return res
-    #         # return [ele.value for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)][::-1]]
-    # else:
-    #     return [ele.value for ele in (ctypes.cast(data,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)]]
+    
+def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False):
+    """Function to convert C binary row to python row
+    """
+    res=[]
+    if num_of_rows > 0:
+        for i in range(abs(num_of_rows)):
+            try:
+                rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop()
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode()[0:rbyte] )
+            except ValueError:
+                res.append(None)
+    else:
+        for i in range(abs(num_of_rows)):
+            try:
+                rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop()
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode()[0:rbyte] )
+            except ValueError:
+                res.append(None)
+    return res
+
+def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False):
+    """Function to convert C nchar row to python row
+    """
+    assert(nbytes is not None)
+    res=[]
+    if num_of_rows >= 0:
+        for i in range(abs(num_of_rows)):
+            try:
+                tmpstr = ctypes.c_char_p(data+nbytes*i+2)
+                res.append( tmpstr.value.decode() )
+            except ValueError:
+                res.append(None)
+    else:
+        for i in range(abs(num_of_rows)):
+            try:
+                res.append( (ctypes.cast(data+nbytes*i+2,  ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value )
+            except ValueError:
+                res.append(None)
+    return res
 
 _CONVERT_FUNC = {
     FieldType.C_BOOL: _crow_bool_to_python,
@@ -128,6 +160,19 @@ _CONVERT_FUNC = {
     FieldType.C_NCHAR : _crow_nchar_to_python
 }
 
+_CONVERT_FUNC_BLOCK = {
+    FieldType.C_BOOL: _crow_bool_to_python,
+    FieldType.C_TINYINT : _crow_tinyint_to_python,
+    FieldType.C_SMALLINT : _crow_smallint_to_python,
+    FieldType.C_INT : _crow_int_to_python,
+    FieldType.C_BIGINT : _crow_bigint_to_python,
+    FieldType.C_FLOAT : _crow_float_to_python,
+    FieldType.C_DOUBLE : _crow_double_to_python,
+    FieldType.C_BINARY: _crow_binary_to_python_block,
+    FieldType.C_TIMESTAMP : _crow_timestamp_to_python, 
+    FieldType.C_NCHAR : _crow_nchar_to_python_block
+}
+
 # Corresponding TAOS_FIELD structure in C
 class TaosField(ctypes.Structure):
     _fields_ = [('name', ctypes.c_char * 65),
@@ -227,8 +272,8 @@ class CTaosInterface(object):
             print('connect to TDengine failed')
             raise ConnectionError("connect to TDengine failed")
             # sys.exit(1)
-        else:
-            print('connect to TDengine success')
+        #else:
+        #    print('connect to TDengine success')
 
         return connection
 
@@ -237,7 +282,7 @@ class CTaosInterface(object):
         '''Close the TDengine handle
         '''
         CTaosInterface.libtaos.taos_close(connection)
-        print('connection is closed')
+        #print('connection is closed')
 
     @staticmethod
     def query(connection, sql):
@@ -310,6 +355,24 @@ class CTaosInterface(object):
 
     @staticmethod
     def fetchBlock(result, fields):
+        pblock = ctypes.c_void_p(0)
+        num_of_rows = CTaosInterface.libtaos.taos_fetch_block(
+            result, ctypes.byref(pblock))
+        if num_of_rows == 0:
+            return None, 0
+        isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO)
+        blocks = [None] * len(fields)
+        fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result)
+        fieldLen = [ele for ele in ctypes.cast(fieldL,  ctypes.POINTER(ctypes.c_int))[:len(fields)]]
+        for i in range(len(fields)):
+            data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i]
+            if fields[i]['type'] not in _CONVERT_FUNC_BLOCK:
+                raise DatabaseError("Invalid data type returned from database")
+            blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro)
+
+        return blocks, abs(num_of_rows)
+    @staticmethod
+    def fetchRow(result, fields):
         pblock = ctypes.c_void_p(0)
         pblock = CTaosInterface.libtaos.taos_fetch_row(result)  
         if pblock : 

src/connector/python/windows/python3/taos/cursor.py

@@ -51,7 +51,7 @@ class TDengineCursor(object):
             raise OperationalError("Invalid use of fetch iterator")
 
         if self._block_rows <= self._block_iter:
-            block, self._block_rows = CTaosInterface.fetchBlock(self._result, self._fields)
+            block, self._block_rows = CTaosInterface.fetchRow(self._result, self._fields)
             if self._block_rows == 0:
                 raise StopIteration
             self._block = list(map(tuple, zip(*block)))
@@ -145,6 +145,24 @@ class TDengineCursor(object):
         if self._result is None or self._fields is None:
             raise OperationalError("Invalid use of fetchall")
 
+        buffer = [[] for i in range(len(self._fields))]
+        self._rowcount = 0
+        while True:
+            block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields)
+            errno = CTaosInterface.libtaos.taos_errno(self._result)
+            if errno != 0:
+                raise ProgrammingError(CTaosInterface.errStr(self._result), errno)
+            if num_of_fields == 0:
+                break
+            self._rowcount += num_of_fields
+            for i in range(len(self._fields)):
+                buffer[i].extend(block[i])
+        return list(map(tuple, zip(*buffer)))
+
+    def fetchall_block(self):
+        if self._result is None or self._fields is None:
+            raise OperationalError("Invalid use of fetchall")
+
         buffer = [[] for i in range(len(self._fields))]
         self._rowcount = 0
         while True:

src/dnode/src/dnodeEps.c

@@ -33,7 +33,7 @@ static void    dnodePrintEps(SDnodeEps *eps);
 
 int32_t dnodeInitEps() {
   pthread_mutex_init(&tsEpsMutex, NULL);
-  tsEpsHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true);
+  tsEpsHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
   dnodeResetEps(NULL);
   int32_t ret = dnodeReadEps();
   if (ret == 0) {

src/dnode/src/dnodeMPeer.c

@@ -162,7 +162,7 @@ static void *dnodeProcessMPeerQueue(void *param) {
       break;
     }
 
-    dDebug("msg:%s will be processed in mpeer queue", taosMsg[pPeerMsg->rpcMsg.msgType]);    
+    dTrace("msg:%s will be processed in mpeer queue", taosMsg[pPeerMsg->rpcMsg.msgType]);    
     int32_t code = mnodeProcessPeerReq(pPeerMsg);    
     dnodeSendRpcMPeerRsp(pPeerMsg, code);    
   }

src/dnode/src/dnodeMRead.c

@@ -168,7 +168,8 @@ static void *dnodeProcessMReadQueue(void *param) {
       break;
     }
 
-    dDebug("%p, msg:%s will be processed in mread queue", pRead->rpcMsg.ahandle, taosMsg[pRead->rpcMsg.msgType]);
+    dTrace("msg:%p, app:%p type:%s will be processed in mread queue", pRead->rpcMsg.ahandle, pRead,
+           taosMsg[pRead->rpcMsg.msgType]);
     int32_t code = mnodeProcessRead(pRead);
     dnodeSendRpcMReadRsp(pRead, code);
   }

src/dnode/src/dnodeMWrite.c

@@ -127,7 +127,7 @@ void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) {
     dnodeSendRedirectMsg(pMsg, true);
   } else {
     SMnodeMsg *pWrite = mnodeCreateMsg(pMsg);
-    dDebug("app:%p:%p, msg:%s is put into mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite,
+    dDebug("msg:%p, app:%p type:%s is put into mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle,
            taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
     taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);
   }
@@ -136,7 +136,7 @@ void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) {
 }
 
 static void dnodeFreeMWriteMsg(SMnodeMsg *pWrite) {
-  dDebug("app:%p:%p, msg:%s is freed from mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite,
+  dDebug("msg:%p, app:%p type:%s is freed from mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle,
          taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
 
   mnodeCleanupMsg(pWrite);
@@ -174,7 +174,7 @@ static void *dnodeProcessMWriteQueue(void *param) {
       break;
     }
 
-    dDebug("app:%p:%p, msg:%s will be processed in mwrite queue", pWrite->rpcMsg.ahandle, pWrite,
+    dTrace("msg:%p, app:%p type:%s will be processed in mwrite queue", pWrite, pWrite->rpcMsg.ahandle,
            taosMsg[pWrite->rpcMsg.msgType]);
 
     int32_t code = mnodeProcessWrite(pWrite);
@@ -188,13 +188,13 @@ void dnodeReprocessMWriteMsg(void *pMsg) {
   SMnodeMsg *pWrite = pMsg;
 
   if (!mnodeIsRunning() || tsMWriteQueue == NULL) {
-    dDebug("app:%p:%p, msg:%s is redirected for mnode not running, retry times:%d", pWrite->rpcMsg.ahandle, pWrite,
+    dDebug("msg:%p, app:%p type:%s is redirected for mnode not running, retry times:%d", pWrite, pWrite->rpcMsg.ahandle,
            taosMsg[pWrite->rpcMsg.msgType], pWrite->retry);
 
     dnodeSendRedirectMsg(pMsg, true);
     dnodeFreeMWriteMsg(pWrite);
   } else {
-    dDebug("app:%p:%p, msg:%s is reput into mwrite queue:%p, retry times:%d", pWrite->rpcMsg.ahandle, pWrite,
+    dDebug("msg:%p, app:%p type:%s is reput into mwrite queue:%p, retry times:%d", pWrite, pWrite->rpcMsg.ahandle,
            taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue, pWrite->retry);
 
     taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);

src/dnode/src/dnodeMain.c

@@ -19,6 +19,7 @@
 #include "tutil.h"
 #include "tconfig.h"
 #include "tglobal.h"
+#include "tfile.h"
 #include "twal.h"
 #include "trpc.h"
 #include "dnode.h"
@@ -55,6 +56,7 @@ typedef struct {
 } SDnodeComponent;
 
 static const SDnodeComponent tsDnodeComponents[] = {
+  {"tfile",     tfInit,              tfCleanup},
   {"rpc",       rpcInit,             rpcCleanup},
   {"storage",   dnodeInitStorage,    dnodeCleanupStorage},
   {"dnodecfg",  dnodeInitCfg,        dnodeCleanupCfg},

src/dnode/src/dnodeMgmt.c

@@ -216,7 +216,7 @@ static void *dnodeProcessMgmtQueue(void *param) {
     }
 
     pMsg = &pMgmt->rpcMsg;
-    dDebug("%p, msg:%p:%s will be processed", pMsg->ahandle, pMgmt, taosMsg[pMsg->msgType]);
+    dDebug("msg:%p, ahandle:%p type:%s will be processed", pMgmt, pMsg->ahandle, taosMsg[pMsg->msgType]);
     if (dnodeProcessMgmtMsgFp[pMsg->msgType]) {
       rsp.code = (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg);
     } else {

src/dnode/src/dnodePeer.c

@@ -151,9 +151,9 @@ void dnodeCleanupClient() {
 }
 
 static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
-  if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) {
+  if (dnodeGetRunStatus() == TSDB_RUN_STATUS_STOPPED) {
     if (pMsg == NULL || pMsg->pCont == NULL) return;
-    dDebug("msg:%p is ignored since dnode not running", pMsg);
+    dDebug("msg:%p is ignored since dnode is stopping", pMsg);
     rpcFreeCont(pMsg->pCont);
     return;
   }

src/dnode/src/dnodeVRead.c

@@ -175,7 +175,7 @@ static void *dnodeProcessReadQueue(void *pWorker) {
       break;
     }
 
-    dDebug("%p, msg:%p:%s will be processed in vread queue, qtype:%d", pRead->rpcAhandle, pRead,
+    dTrace("msg:%p, app:%p type:%s will be processed in vread queue, qtype:%d", pRead, pRead->rpcAhandle,
            taosMsg[pRead->msgType], qtype);
 
     int32_t code = vnodeProcessRead(pVnode, pRead);

src/dnode/src/dnodeVWrite.c

@@ -176,7 +176,7 @@ void dnodeSendRpcVWriteRsp(void *pVnode, void *wparam, int32_t code) {
   if (count <= 1) return;
 
   SRpcMsg rpcRsp = {
-    .handle  = pWrite->rpcHandle,
+    .handle  = pWrite->rpcMsg.handle,
     .pCont   = pWrite->rspRet.rsp,
     .contLen = pWrite->rspRet.len,
     .code    = pWrite->code,
@@ -205,8 +205,8 @@ static void *dnodeProcessVWriteQueue(void *wparam) {
     bool forceFsync = false;
     for (int32_t i = 0; i < numOfMsgs; ++i) {
       taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite);
-      dTrace("%p, msg:%p:%s will be processed in vwrite queue, qtype:%s hver:%" PRIu64, pWrite->rpcAhandle, pWrite,
-             taosMsg[pWrite->pHead->msgType], qtypeStr[qtype], pWrite->pHead->version);
+      dTrace("msg:%p, app:%p type:%s will be processed in vwrite queue, qtype:%s hver:%" PRIu64, pWrite,
+             pWrite->rpcMsg.ahandle, taosMsg[pWrite->pHead->msgType], qtypeStr[qtype], pWrite->pHead->version);
 
       pWrite->code = vnodeProcessWrite(pVnode, pWrite->pHead, qtype, &pWrite->rspRet);
       if (pWrite->code <= 0) pWrite->processedCount = 1;

src/inc/taoserror.h

@@ -205,6 +205,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VRESION_FILE,     0, 0x050A, "Invalid ve
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FULL,                  0, 0x050B, "Vnode memory is full because commit failed")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED,               0, 0x0511, "Database suspended")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH,            0, 0x0512, "Write operation denied")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_SYNCING,                  0, 0x0513, "Database is syncing")
 
 // tsdb
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID,         0, 0x0600, "Invalid table ID")
@@ -259,6 +260,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED,            0, 0x080B, "CPU cores
 TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CONFIG,           0, 0x0900, "Invalid Sync Configuration")
 TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED,              0, 0x0901, "Sync module not enabled")
 TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_VERSION,          0, 0x0902, "Invalid Sync version")
+TAOS_DEFINE_ERROR(TSDB_CODE_SYN_CONFIRM_EXPIRED,          0, 0x0903, "Sync confirm expired")
+TAOS_DEFINE_ERROR(TSDB_CODE_SYN_VND_COMMITING,            0, 0x0904, "Vnode is commiting")
+TAOS_DEFINE_ERROR(TSDB_CODE_SYN_FILE_CHNAGED,             0, 0x0905, "Vnode file is changed")
+TAOS_DEFINE_ERROR(TSDB_CODE_SYN_APP_ERROR,                0, 0x1000, "Unexpected generic error in sync")
 
 // wal
 TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR,                0, 0x1000, "Unexpected generic error in wal")
@@ -365,7 +370,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TOO_LONG,   0, 0x11A4, "tag value
 TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_NULL,           0, 0x11A5, "value not find")
 TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_TYPE,           0, 0x11A6, "value type should be boolean, number or string")
 
-
+// odbc
 TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_OOM,                     0, 0x2100, "out of memory")
 TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_CHAR_NOT_NUM,       0, 0x2101, "convertion not a valid literal input")
 TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_UNDEF,              0, 0x2102, "convertion undefined")
@@ -389,7 +394,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_BAD_SEQ,        0, 0x2113, "src bad se
 TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_INCOMPLETE,     0, 0x2114, "src incomplete")
 TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_GENERAL,        0, 0x2115, "src general")
 
-
 #ifdef TAOS_ERROR_C
 };
 #endif

src/inc/tsync.h

@@ -23,19 +23,19 @@ extern "C" {
 #define TAOS_SYNC_MAX_REPLICA 5
 #define TAOS_SYNC_MAX_INDEX   0x7FFFFFFF
 
-typedef enum _TAOS_SYNC_ROLE {
-  TAOS_SYNC_ROLE_OFFLINE,
-  TAOS_SYNC_ROLE_UNSYNCED,
-  TAOS_SYNC_ROLE_SYNCING,
-  TAOS_SYNC_ROLE_SLAVE,
-  TAOS_SYNC_ROLE_MASTER,
+typedef enum {
+  TAOS_SYNC_ROLE_OFFLINE  = 0,
+  TAOS_SYNC_ROLE_UNSYNCED = 1,
+  TAOS_SYNC_ROLE_SYNCING  = 2,
+  TAOS_SYNC_ROLE_SLAVE    = 3,
+  TAOS_SYNC_ROLE_MASTER   = 4
 } ESyncRole;
 
-typedef enum _TAOS_SYNC_STATUS {
-  TAOS_SYNC_STATUS_INIT,
-  TAOS_SYNC_STATUS_START,
-  TAOS_SYNC_STATUS_FILE,
-  TAOS_SYNC_STATUS_CACHE,
+typedef enum {
+  TAOS_SYNC_STATUS_INIT  = 0,
+  TAOS_SYNC_STATUS_START = 1,
+  TAOS_SYNC_STATUS_FILE  = 2,
+  TAOS_SYNC_STATUS_CACHE = 3
 } ESyncStatus;
 
 typedef struct {
@@ -64,33 +64,35 @@ typedef struct {
   if name is provided(name[0] is not zero), get the named file at the specified index. If not there, return
   zero. If it is there, set the size to file size, and return file magic number. Index shall not be updated.
 */
-typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); 
+typedef uint32_t (*FGetFileInfo)(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); 
 
 // get the wal file from index or after
 // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file
-typedef int32_t  (*FGetWalInfo)(void *ahandle, char *fileName, int64_t *fileId); 
+typedef int32_t  (*FGetWalInfo)(int32_t vgId, char *fileName, int64_t *fileId); 
  
 // when a forward pkt is received, call this to handle data
-typedef int32_t  (*FWriteToCache)(void *ahandle, void *pHead, int32_t qtype, void *pMsg);
+typedef int32_t  (*FWriteToCache)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg);
 
 // when forward is confirmed by peer, master call this API to notify app
-typedef void     (*FConfirmForward)(void *ahandle, void *mhandle, int32_t code);
+typedef void     (*FConfirmForward)(int32_t vgId, void *mhandle, int32_t code);
 
 // when role is changed, call this to notify app
-typedef void     (*FNotifyRole)(void *ahandle, int8_t role);
+typedef void     (*FNotifyRole)(int32_t vgId, int8_t role);
 
 // if a number of retrieving data failed, call this to start flow control 
-typedef void     (*FNotifyFlowCtrl)(void *ahandle, int32_t mseconds);
+typedef void     (*FNotifyFlowCtrl)(int32_t vgId, int32_t level);
 
 // when data file is synced successfully, notity app
-typedef int32_t  (*FNotifyFileSynced)(void *ahandle, uint64_t fversion);
+typedef int32_t  (*FNotifyFileSynced)(int32_t vgId, uint64_t fversion);
+
+// get file version
+typedef int32_t  (*FGetFileVersion)(int32_t vgId, uint64_t *fver);
 
 typedef struct {
   int32_t  vgId;       // vgroup ID
   uint64_t version;    // initial version
   SSyncCfg syncCfg;    // configuration from mgmt
-  char     path[128];  // path to the file
-  void *   ahandle;    // handle provided by APP
+  char     path[TSDB_FILENAME_LEN];  // path to the file
   FGetFileInfo      getFileInfo;
   FGetWalInfo       getWalInfo;
   FWriteToCache     writeToCache;
@@ -98,6 +100,7 @@ typedef struct {
   FNotifyRole       notifyRole;
   FNotifyFlowCtrl   notifyFlowCtrl;
   FNotifyFileSynced notifyFileSynced;
+  FGetFileVersion   getFileVersion;
 } SSyncInfo;
 
 typedef void *tsync_h;

src/inc/twal.h

@@ -53,7 +53,6 @@ typedef int32_t FWalWrite(void *ahandle, void *pHead, int32_t qtype, void *pMsg)
 
 int32_t  walInit();
 void     walCleanUp();
-
 twalh    walOpen(char *path, SWalCfg *pCfg);
 int32_t  walAlter(twalh pWal, SWalCfg *pCfg);
 void     walStop(twalh);

src/inc/vnode.h

@@ -20,6 +20,7 @@
 extern "C" {
 #endif
 
+#include "trpc.h"
 #include "twal.h"
 
 typedef enum _VN_STATUS {
@@ -51,8 +52,9 @@ typedef struct {
 typedef struct {
   int32_t  code;
   int32_t  processedCount;
-  void *   rpcHandle;
-  void *   rpcAhandle;
+  int32_t  qtype;
+  void *   pVnode;
+  SRpcMsg  rpcMsg;
   SRspRet  rspRet;
   char     reserveForSync[16];
   SWalHead pHead[];

src/kit/taosdump/taosdump.c

@@ -2146,7 +2146,7 @@ int taosDumpInOneFile(TAOS     * taos, FILE* fp, char* fcharset, char* encode, c
   char *    line     = NULL;
   size_t    line_len = 0;
 
-  cmd  = (char *)malloc(COMMAND_SIZE);
+  cmd  = (char *)malloc(TSDB_MAX_ALLOWED_SQL_LEN);
   if (cmd == NULL) {
     fprintf(stderr, "failed to allocate memory\n");
     return -1;
@@ -2155,7 +2155,7 @@ int taosDumpInOneFile(TAOS     * taos, FILE* fp, char* fcharset, char* encode, c
   int lineNo = 0;
   while ((read_len = getline(&line, &line_len, fp)) != -1) {
     ++lineNo;
-    if (read_len >= COMMAND_SIZE) continue;
+    if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue;
     line[--read_len] = '\0';
 
     //if (read_len == 0 || isCommentLine(line)) {  // line starts with #
@@ -2176,7 +2176,7 @@ int taosDumpInOneFile(TAOS     * taos, FILE* fp, char* fcharset, char* encode, c
       fprintf(stderr, "error sql: linenu:%d, file:%s\n", lineNo, fileName);
     }
 
-    memset(cmd, 0, COMMAND_SIZE);
+    memset(cmd, 0, TSDB_MAX_ALLOWED_SQL_LEN);
     cmd_len = 0;
   }
 

src/mnode/inc/mnodeAcct.h

@@ -27,6 +27,7 @@ void    mnodeCleanupAccts();
 void    mnodeGetStatOfAllAcct(SAcctInfo* pAcctInfo);
 void *  mnodeGetAcct(char *acctName);
 void *  mnodeGetNextAcct(void *pIter, SAcctObj **pAcct);
+void    mnodeCancelGetNextAcct(void *pIter);
 void    mnodeIncAcctRef(SAcctObj *pAcct);
 void    mnodeDecAcctRef(SAcctObj *pAcct);
 void    mnodeAddDbToAcct(SAcctObj *pAcct, SDbObj *pDb);

src/mnode/inc/mnodeDb.h

@@ -34,6 +34,7 @@ int64_t mnodeGetDbNum();
 SDbObj *mnodeGetDb(char *db);
 SDbObj *mnodeGetDbByTableId(char *db);
 void *  mnodeGetNextDb(void *pIter, SDbObj **pDb);
+void    mnodeCancelGetNextDb(void *pIter);
 void    mnodeIncDbRef(SDbObj *pDb);
 void    mnodeDecDbRef(SDbObj *pDb);
 bool    mnodeCheckIsMonitorDB(char *db, char *monitordb);

src/mnode/inc/mnodeDnode.h

@@ -65,6 +65,7 @@ int32_t mnodeGetDnodesNum();
 int32_t mnodeGetOnlinDnodesCpuCoreNum();
 int32_t mnodeGetOnlineDnodesNum();
 void *  mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode);
+void    mnodeCancelGetNextDnode(void *pIter);
 void    mnodeIncDnodeRef(SDnodeObj *pDnode);
 void    mnodeDecDnodeRef(SDnodeObj *pDnode);
 void *  mnodeGetDnode(int32_t dnodeId);

src/mnode/inc/mnodeMnode.h

@@ -38,6 +38,7 @@ void    mnodeDropMnodeLocal(int32_t dnodeId);
 void *  mnodeGetMnode(int32_t mnodeId);
 int32_t mnodeGetMnodesNum();
 void *  mnodeGetNextMnode(void *pIter, struct SMnodeObj **pMnode);
+void    mnodeCancelGetNextMnode(void *pIter);
 void    mnodeIncMnodeRef(struct SMnodeObj *pMnode);
 void    mnodeDecMnodeRef(struct SMnodeObj *pMnode);
 

src/mnode/inc/mnodeSdb.h

@@ -20,7 +20,8 @@
 extern "C" {
 #endif
 
-struct SMnodeMsg;
+#include "mnode.h"
+#include "twal.h"
 
 typedef enum {
   SDB_TABLE_CLUSTER = 0,
@@ -36,44 +37,46 @@ typedef enum {
 } ESdbTable;
 
 typedef enum {
-  SDB_KEY_STRING, 
-  SDB_KEY_INT,
-  SDB_KEY_AUTO,
-  SDB_KEY_VAR_STRING,
+  SDB_KEY_STRING     = 0, 
+  SDB_KEY_INT        = 1,
+  SDB_KEY_AUTO       = 2,
+  SDB_KEY_VAR_STRING = 3,
 } ESdbKey;
 
 typedef enum {
-  SDB_OPER_GLOBAL,
-  SDB_OPER_LOCAL
+  SDB_OPER_GLOBAL = 0,
+  SDB_OPER_LOCAL  = 1
 } ESdbOper;
 
-typedef struct SSdbOper {
+typedef struct SSdbRow {
   ESdbOper   type;
-  int32_t  rowSize;
-  int32_t  retCode; // for callback in sdb queue
   int32_t    processedCount;  // for sync fwd callback
-  int32_t  (*reqFp)(struct SMnodeMsg *pMsg);
-  int32_t  (*writeCb)(struct SMnodeMsg *pMsg, int32_t code);
-  void *   table;
-  void *   pObj;
+  int32_t    code;            // for callback in sdb queue
+  int32_t    rowSize;
   void *     rowData;
-  struct SMnodeMsg *pMsg;
-} SSdbOper;
+  void *     pObj;
+  void *     pTable;
+  SMnodeMsg *pMsg;
+  int32_t  (*fpReq)(SMnodeMsg *pMsg);
+  int32_t  (*fpRsp)(SMnodeMsg *pMsg, int32_t code);
+  char       reserveForSync[16];
+  SWalHead   pHead[];
+} SSdbRow;
 
 typedef struct {
-  char   *tableName;
+  char *    name;
   int32_t   hashSessions;
   int32_t   maxRowSize;
   int32_t   refCountPos;
-  ESdbTable tableId;
+  ESdbTable id;
   ESdbKey   keyType;
-  int32_t (*insertFp)(SSdbOper *pOper);
-  int32_t (*deleteFp)(SSdbOper *pOper);
-  int32_t (*updateFp)(SSdbOper *pOper);
-  int32_t (*encodeFp)(SSdbOper *pOper);
-  int32_t (*decodeFp)(SSdbOper *pDesc);  
-  int32_t (*destroyFp)(SSdbOper *pDesc);
-  int32_t (*restoredFp)();
+  int32_t (*fpInsert)(SSdbRow *pRow);
+  int32_t (*fpDelete)(SSdbRow *pRow);
+  int32_t (*fpUpdate)(SSdbRow *pRow);
+  int32_t (*fpEncode)(SSdbRow *pRow);
+  int32_t (*fpDecode)(SSdbRow *pRow);  
+  int32_t (*fpDestroy)(SSdbRow *pRow);
+  int32_t (*fpRestored)();
 } SSdbTableDesc;
 
 int32_t sdbInit();
@@ -84,20 +87,20 @@ bool    sdbIsMaster();
 bool    sdbIsServing();
 void    sdbUpdateMnodeRoles();
 
-int32_t sdbInsertRow(SSdbOper *pOper);
-int32_t sdbDeleteRow(SSdbOper *pOper);
-int32_t sdbUpdateRow(SSdbOper *pOper);
-int32_t sdbInsertRowImp(SSdbOper *pOper);
+int32_t sdbInsertRow(SSdbRow *pRow);
+int32_t sdbDeleteRow(SSdbRow *pRow);
+int32_t sdbUpdateRow(SSdbRow *pRow);
+int32_t sdbInsertRowToQueue(SSdbRow *pRow);
 
-void    *sdbGetRow(void *handle, void *key);
-void    *sdbFetchRow(void *handle, void *pIter, void **ppRow);
-void     sdbFreeIter(void *pIter);
-void     sdbIncRef(void *thandle, void *pRow);
-void     sdbDecRef(void *thandle, void *pRow);
-int64_t  sdbGetNumOfRows(void *handle);
-int32_t  sdbGetId(void *handle);
+void *   sdbGetRow(void *pTable, void *key);
+void *   sdbFetchRow(void *pTable, void *pIter, void **ppRow);
+void     sdbFreeIter(void *pTable, void *pIter);
+void     sdbIncRef(void *pTable, void *pRow);
+void     sdbDecRef(void *pTable, void *pRow);
+int64_t  sdbGetNumOfRows(void *pTable);
+int32_t  sdbGetId(void *pTable);
 uint64_t sdbGetVersion();
-bool     sdbCheckRowDeleted(void *thandle, void *pRow);
+bool     sdbCheckRowDeleted(void *pTable, void *pRow);
 
 #ifdef __cplusplus
 }

src/mnode/inc/mnodeShow.h

@@ -26,8 +26,10 @@ void    mnodeCleanUpShow();
 
 typedef int32_t (*SShowMetaFp)(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 typedef int32_t (*SShowRetrieveFp)(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+typedef void    (*SShowFreeIterFp)(void *pIter);
 void mnodeAddShowMetaHandle(uint8_t showType, SShowMetaFp fp);
 void mnodeAddShowRetrieveHandle(uint8_t showType, SShowRetrieveFp fp);
+void mnodeAddShowFreeIterHandle(uint8_t msgType, SShowFreeIterFp fp);
 void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow);
 
 #ifdef __cplusplus

src/mnode/inc/mnodeTable.h

@@ -31,6 +31,8 @@ void    mnodeIncTableRef(void *pTable);
 void    mnodeDecTableRef(void *pTable);
 void *  mnodeGetNextChildTable(void *pIter, SCTableObj **pTable);
 void *  mnodeGetNextSuperTable(void *pIter, SSTableObj **pTable);
+void    mnodeCancelGetNextChildTable(void *pIter);
+void    mnodeCancelGetNextSuperTable(void *pIter);
 void    mnodeDropAllChildTables(SDbObj *pDropDb);
 void    mnodeDropAllSuperTables(SDbObj *pDropDb);
 void    mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup);

src/mnode/inc/mnodeUser.h

@@ -25,6 +25,7 @@ int32_t   mnodeInitUsers();
 void      mnodeCleanupUsers();
 SUserObj *mnodeGetUser(char *name);
 void *    mnodeGetNextUser(void *pIter, SUserObj **pUser);
+void      mnodeCancelGetNextUser(void *pIter);
 void      mnodeIncUserRef(SUserObj *pUser);
 void      mnodeDecUserRef(SUserObj *pUser);
 SUserObj *mnodeGetUserFromConn(void *pConn);

src/mnode/inc/mnodeVgroup.h

@@ -34,6 +34,7 @@ void    mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode);
 //void  mnodeUpdateAllDbVgroups(SDbObj *pAlterDb);
 
 void *  mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup);
+void    mnodeCancelGetNextVgroup(void *pIter);
 void    mnodeUpdateVgroup(SVgObj *pVgroup);
 void    mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload);
 void    mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t openVnodes);

src/mnode/src/mnodeAcct.c

@@ -16,6 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 #include "taoserror.h"
+#include "tglobal.h"
 #include "dnode.h"
 #include "mnodeDef.h"
 #include "mnodeInt.h"
@@ -25,36 +26,34 @@
 #include "mnodeUser.h"
 #include "mnodeVgroup.h"
 
-#include "tglobal.h"
-
 void *  tsAcctSdb = NULL;
 static int32_t tsAcctUpdateSize;
 static int32_t mnodeCreateRootAcct();
 
-static int32_t mnodeAcctActionDestroy(SSdbOper *pOper) {
-  SAcctObj *pAcct = pOper->pObj;
+static int32_t mnodeAcctActionDestroy(SSdbRow *pRow) {
+  SAcctObj *pAcct = pRow->pObj;
   pthread_mutex_destroy(&pAcct->mutex);
-  tfree(pOper->pObj);
+  tfree(pRow->pObj);
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeAcctActionInsert(SSdbOper *pOper) {
-  SAcctObj *pAcct = pOper->pObj;
+static int32_t mnodeAcctActionInsert(SSdbRow *pRow) {
+  SAcctObj *pAcct = pRow->pObj;
   memset(&pAcct->acctInfo, 0, sizeof(SAcctInfo));
   pAcct->acctInfo.accessState = TSDB_VN_ALL_ACCCESS;
   pthread_mutex_init(&pAcct->mutex, NULL);
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeAcctActionDelete(SSdbOper *pOper) {
-  SAcctObj *pAcct = pOper->pObj;
+static int32_t mnodeAcctActionDelete(SSdbRow *pRow) {
+  SAcctObj *pAcct = pRow->pObj;
   mnodeDropAllUsers(pAcct);
   mnodeDropAllDbs(pAcct);
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeAcctActionUpdate(SSdbOper *pOper) {
-  SAcctObj *pAcct = pOper->pObj;
+static int32_t mnodeAcctActionUpdate(SSdbRow *pRow) {
+  SAcctObj *pAcct = pRow->pObj;
   SAcctObj *pSaved = mnodeGetAcct(pAcct->user);
   if (pAcct != pSaved) {
     memcpy(pSaved, pAcct, tsAcctUpdateSize);
@@ -64,19 +63,19 @@ static int32_t mnodeAcctActionUpdate(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeAcctActionEncode(SSdbOper *pOper) {
-  SAcctObj *pAcct = pOper->pObj;
-  memcpy(pOper->rowData, pAcct, tsAcctUpdateSize);
-  pOper->rowSize = tsAcctUpdateSize;
+static int32_t mnodeAcctActionEncode(SSdbRow *pRow) {
+  SAcctObj *pAcct = pRow->pObj;
+  memcpy(pRow->rowData, pAcct, tsAcctUpdateSize);
+  pRow->rowSize = tsAcctUpdateSize;
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeAcctActionDecode(SSdbOper *pOper) {
+static int32_t mnodeAcctActionDecode(SSdbRow *pRow) {
   SAcctObj *pAcct = (SAcctObj *) calloc(1, sizeof(SAcctObj));
   if (pAcct == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
 
-  memcpy(pAcct, pOper->rowData, tsAcctUpdateSize);
-  pOper->pObj = pAcct;
+  memcpy(pAcct, pRow->rowData, tsAcctUpdateSize);
+  pRow->pObj = pAcct;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -99,29 +98,29 @@ int32_t mnodeInitAccts() {
   SAcctObj tObj;
   tsAcctUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
 
-  SSdbTableDesc tableDesc = {
-    .tableId      = SDB_TABLE_ACCOUNT,
-    .tableName    = "accounts",
+  SSdbTableDesc desc = {
+    .id           = SDB_TABLE_ACCOUNT,
+    .name         = "accounts",
     .hashSessions = TSDB_DEFAULT_ACCOUNTS_HASH_SIZE,
     .maxRowSize   = tsAcctUpdateSize,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_STRING,
-    .insertFp     = mnodeAcctActionInsert,
-    .deleteFp     = mnodeAcctActionDelete,
-    .updateFp     = mnodeAcctActionUpdate,
-    .encodeFp     = mnodeAcctActionEncode,
-    .decodeFp     = mnodeAcctActionDecode,
-    .destroyFp    = mnodeAcctActionDestroy,
-    .restoredFp   = mnodeAcctActionRestored
+    .fpInsert     = mnodeAcctActionInsert,
+    .fpDelete     = mnodeAcctActionDelete,
+    .fpUpdate     = mnodeAcctActionUpdate,
+    .fpEncode     = mnodeAcctActionEncode,
+    .fpDecode     = mnodeAcctActionDecode,
+    .fpDestroy    = mnodeAcctActionDestroy,
+    .fpRestored   = mnodeAcctActionRestored
   };
 
-  tsAcctSdb = sdbOpenTable(&tableDesc);
+  tsAcctSdb = sdbOpenTable(&desc);
   if (tsAcctSdb == NULL) {
-    mError("table:%s, failed to create hash", tableDesc.tableName);
+    mError("table:%s, failed to create hash", desc.name);
     return -1;
   }
 
-  mDebug("table:%s, hash is created", tableDesc.tableName);
+  mDebug("table:%s, hash is created", desc.name);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -145,7 +144,6 @@ void mnodeGetStatOfAllAcct(SAcctInfo* pAcctInfo) {
     pAcctInfo->numOfTimeSeries += pAcct->acctInfo.numOfTimeSeries;
     mnodeDecAcctRef(pAcct);
   }
-  sdbFreeIter(pIter);
 
   SVgObj *pVgroup = NULL;
   pIter = NULL;
@@ -159,7 +157,6 @@ void mnodeGetStatOfAllAcct(SAcctInfo* pAcctInfo) {
     pAcctInfo->totalPoints += pVgroup->pointsWritten;
     mnodeDecVgroupRef(pVgroup);
   }
-  sdbFreeIter(pIter);
 }
 
 void *mnodeGetAcct(char *name) {
@@ -170,6 +167,10 @@ void *mnodeGetNextAcct(void *pIter, SAcctObj **pAcct) {
   return sdbFetchRow(tsAcctSdb, pIter, (void **)pAcct); 
 }
 
+void mnodeCancelGetNextAcct(void *pIter) {
+  sdbFreeIter(tsAcctSdb, pIter);
+}
+
 void mnodeIncAcctRef(SAcctObj *pAcct) {
   sdbIncRef(tsAcctSdb, pAcct);
 }
@@ -226,13 +227,13 @@ static int32_t mnodeCreateRootAcct() {
   pAcct->acctId = sdbGetId(tsAcctSdb);
   pAcct->createdTime = taosGetTimestampMs();
 
-  SSdbOper oper = {
+  SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,
-    .table = tsAcctSdb,
+    .pTable = tsAcctSdb,
     .pObj   = pAcct,
   };
 
-  return sdbInsertRow(&oper);
+  return sdbInsertRow(&row);
 }
 
 #ifndef _ACCT

src/mnode/src/mnodeCluster.c

@@ -31,37 +31,38 @@ static int32_t mnodeCreateCluster();
 
 static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+static void    mnodeCancelGetNextCluster(void *pIter);
 
-static int32_t mnodeClusterActionDestroy(SSdbOper *pOper) {
-  tfree(pOper->pObj);
+static int32_t mnodeClusterActionDestroy(SSdbRow *pRow) {
+  tfree(pRow->pObj);
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeClusterActionInsert(SSdbOper *pOper) {
+static int32_t mnodeClusterActionInsert(SSdbRow *pRow) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeClusterActionDelete(SSdbOper *pOper) {
+static int32_t mnodeClusterActionDelete(SSdbRow *pRow) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeClusterActionUpdate(SSdbOper *pOper) {
+static int32_t mnodeClusterActionUpdate(SSdbRow *pRow) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeClusterActionEncode(SSdbOper *pOper) {
-  SClusterObj *pCluster = pOper->pObj;
-  memcpy(pOper->rowData, pCluster, tsClusterUpdateSize);
-  pOper->rowSize = tsClusterUpdateSize;
+static int32_t mnodeClusterActionEncode(SSdbRow *pRow) {
+  SClusterObj *pCluster = pRow->pObj;
+  memcpy(pRow->rowData, pCluster, tsClusterUpdateSize);
+  pRow->rowSize = tsClusterUpdateSize;
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeClusterActionDecode(SSdbOper *pOper) {
+static int32_t mnodeClusterActionDecode(SSdbRow *pRow) {
   SClusterObj *pCluster = (SClusterObj *) calloc(1, sizeof(SClusterObj));
   if (pCluster == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
 
-  memcpy(pCluster, pOper->rowData, tsClusterUpdateSize);
-  pOper->pObj = pCluster;
+  memcpy(pCluster, pRow->rowData, tsClusterUpdateSize);
+  pRow->pObj = pCluster;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -84,32 +85,33 @@ int32_t mnodeInitCluster() {
   SClusterObj tObj;
   tsClusterUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
 
-  SSdbTableDesc tableDesc = {
-    .tableId      = SDB_TABLE_CLUSTER,
-    .tableName    = "cluster",
+  SSdbTableDesc desc = {
+    .id           = SDB_TABLE_CLUSTER,
+    .name         = "cluster",
     .hashSessions = TSDB_DEFAULT_CLUSTER_HASH_SIZE,
     .maxRowSize   = tsClusterUpdateSize,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_STRING,
-    .insertFp     = mnodeClusterActionInsert,
-    .deleteFp     = mnodeClusterActionDelete,
-    .updateFp     = mnodeClusterActionUpdate,
-    .encodeFp     = mnodeClusterActionEncode,
-    .decodeFp     = mnodeClusterActionDecode,
-    .destroyFp    = mnodeClusterActionDestroy,
-    .restoredFp   = mnodeClusterActionRestored
+    .fpInsert     = mnodeClusterActionInsert,
+    .fpDelete     = mnodeClusterActionDelete,
+    .fpUpdate     = mnodeClusterActionUpdate,
+    .fpEncode     = mnodeClusterActionEncode,
+    .fpDecode     = mnodeClusterActionDecode,
+    .fpDestroy    = mnodeClusterActionDestroy,
+    .fpRestored   = mnodeClusterActionRestored
   };
 
-  tsClusterSdb = sdbOpenTable(&tableDesc);
+  tsClusterSdb = sdbOpenTable(&desc);
   if (tsClusterSdb == NULL) {
-    mError("table:%s, failed to create hash", tableDesc.tableName);
+    mError("table:%s, failed to create hash", desc.name);
     return -1;
   }
 
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeGetClusterMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeRetrieveClusters);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeCancelGetNextCluster);
 
-  mDebug("table:%s, hash is created", tableDesc.tableName);
+  mDebug("table:%s, hash is created", desc.name);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -122,6 +124,10 @@ void *mnodeGetNextCluster(void *pIter, SClusterObj **pCluster) {
   return sdbFetchRow(tsClusterSdb, pIter, (void **)pCluster); 
 }
 
+void mnodeCancelGetNextCluster(void *pIter) {
+  sdbFreeIter(tsClusterSdb, pIter);
+}
+
 void mnodeIncClusterRef(SClusterObj *pCluster) {
   sdbIncRef(tsClusterSdb, pCluster);
 }
@@ -145,13 +151,13 @@ static int32_t mnodeCreateCluster() {
     mDebug("uid is %s", pCluster->uid);
   }
 
-  SSdbOper oper = {
+  SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,
-    .table = tsClusterSdb,
+    .pTable = tsClusterSdb,
     .pObj   = pCluster,
   };
 
-  return sdbInsertRow(&oper);
+  return sdbInsertRow(&row);
 }
 
 const char* mnodeGetClusterId() {
@@ -167,7 +173,7 @@ void mnodeUpdateClusterId() {
   }
 
   mnodeDecClusterRef(pCluster);
-  sdbFreeIter(pIter);
+  mnodeCancelGetNextCluster(pIter);
 }
 
 static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {

src/mnode/src/mnodeDb.c

@@ -56,8 +56,8 @@ static void mnodeDestroyDb(SDbObj *pDb) {
   tfree(pDb);
 }
 
-static int32_t mnodeDbActionDestroy(SSdbOper *pOper) {
-  mnodeDestroyDb(pOper->pObj);
+static int32_t mnodeDbActionDestroy(SSdbRow *pRow) {
+  mnodeDestroyDb(pRow->pObj);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -65,8 +65,8 @@ int64_t mnodeGetDbNum() {
   return sdbGetNumOfRows(tsDbSdb);
 }
 
-static int32_t mnodeDbActionInsert(SSdbOper *pOper) {
-  SDbObj *pDb = pOper->pObj;
+static int32_t mnodeDbActionInsert(SSdbRow *pRow) {
+  SDbObj *pDb = pRow->pObj;
   SAcctObj *pAcct = mnodeGetAcct(pDb->acct);
 
   pthread_mutex_init(&pDb->mutex, NULL);
@@ -91,8 +91,8 @@ static int32_t mnodeDbActionInsert(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeDbActionDelete(SSdbOper *pOper) {
-  SDbObj *pDb = pOper->pObj;
+static int32_t mnodeDbActionDelete(SSdbRow *pRow) {
+  SDbObj *pDb = pRow->pObj;
   SAcctObj *pAcct = mnodeGetAcct(pDb->acct);
 
   mnodeDropAllChildTables(pDb);
@@ -107,11 +107,11 @@ static int32_t mnodeDbActionDelete(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeDbActionUpdate(SSdbOper *pOper) {
-  SDbObj *pNew = pOper->pObj;
+static int32_t mnodeDbActionUpdate(SSdbRow *pRow) {
+  SDbObj *pNew = pRow->pObj;
   SDbObj *pDb = mnodeGetDb(pNew->name);
   if (pDb != NULL && pNew != pDb) {
-    memcpy(pDb, pNew, pOper->rowSize);
+    memcpy(pDb, pNew, pRow->rowSize);
     free(pNew->vgList);
     free(pNew);
   }
@@ -120,19 +120,19 @@ static int32_t mnodeDbActionUpdate(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeDbActionEncode(SSdbOper *pOper) {
-  SDbObj *pDb = pOper->pObj;
-  memcpy(pOper->rowData, pDb, tsDbUpdateSize);
-  pOper->rowSize = tsDbUpdateSize;
+static int32_t mnodeDbActionEncode(SSdbRow *pRow) {
+  SDbObj *pDb = pRow->pObj;
+  memcpy(pRow->rowData, pDb, tsDbUpdateSize);
+  pRow->rowSize = tsDbUpdateSize;
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeDbActionDecode(SSdbOper *pOper) {
+static int32_t mnodeDbActionDecode(SSdbRow *pRow) {
   SDbObj *pDb = (SDbObj *) calloc(1, sizeof(SDbObj));
   if (pDb == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
   
-  memcpy(pDb, pOper->rowData, tsDbUpdateSize);
-  pOper->pObj = pDb;
+  memcpy(pDb, pRow->rowData, tsDbUpdateSize);
+  pRow->pObj = pDb;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -144,23 +144,23 @@ int32_t mnodeInitDbs() {
   SDbObj tObj;
   tsDbUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
 
-  SSdbTableDesc tableDesc = {
-    .tableId      = SDB_TABLE_DB,
-    .tableName    = "dbs",
+  SSdbTableDesc desc = {
+    .id           = SDB_TABLE_DB,
+    .name         = "dbs",
     .hashSessions = TSDB_DEFAULT_DBS_HASH_SIZE,
     .maxRowSize   = tsDbUpdateSize,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_STRING,
-    .insertFp     = mnodeDbActionInsert,
-    .deleteFp     = mnodeDbActionDelete,
-    .updateFp     = mnodeDbActionUpdate,
-    .encodeFp     = mnodeDbActionEncode,
-    .decodeFp     = mnodeDbActionDecode,
-    .destroyFp    = mnodeDbActionDestroy,
-    .restoredFp   = mnodeDbActionRestored
+    .fpInsert     = mnodeDbActionInsert,
+    .fpDelete     = mnodeDbActionDelete,
+    .fpUpdate     = mnodeDbActionUpdate,
+    .fpEncode     = mnodeDbActionEncode,
+    .fpDecode     = mnodeDbActionDecode,
+    .fpDestroy    = mnodeDbActionDestroy,
+    .fpRestored   = mnodeDbActionRestored
   };
 
-  tsDbSdb = sdbOpenTable(&tableDesc);
+  tsDbSdb = sdbOpenTable(&desc);
   if (tsDbSdb == NULL) {
     mError("failed to init db data");
     return -1;
@@ -171,6 +171,7 @@ int32_t mnodeInitDbs() {
   mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_DROP_DB, mnodeProcessDropDbMsg);
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_DB, mnodeGetDbMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_DB, mnodeRetrieveDbs);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_DB, mnodeCancelGetNextDb);
   
   mDebug("table:dbs table is created");
   return 0;
@@ -180,6 +181,10 @@ void *mnodeGetNextDb(void *pIter, SDbObj **pDb) {
   return sdbFetchRow(tsDbSdb, pIter, (void **)pDb);
 }
 
+void mnodeCancelGetNextDb(void *pIter) {
+  sdbFreeIter(tsDbSdb, pIter);
+}
+
 SDbObj *mnodeGetDb(char *db) {
   return (SDbObj *)sdbGetRow(tsDbSdb, db);
 }
@@ -412,16 +417,16 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg *
   pMsg->pDb = pDb;
   mnodeIncDbRef(pDb);
 
-  SSdbOper oper = {
+  SSdbRow row = {
     .type     = SDB_OPER_GLOBAL,
-    .table   = tsDbSdb,
+    .pTable   = tsDbSdb,
     .pObj     = pDb,
     .rowSize  = sizeof(SDbObj),
     .pMsg     = pMsg,
-    .writeCb = mnodeCreateDbCb
+    .fpRsp    = mnodeCreateDbCb
   };
 
-  code = sdbInsertRow(&oper);
+  code = sdbInsertRow(&row);
   if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mError("db:%s, failed to create, reason:%s", pDb->name, tstrerror(code));
     pMsg->pDb = NULL;
@@ -440,8 +445,8 @@ bool mnodeCheckIsMonitorDB(char *db, char *monitordb) {
 }
 
 #if 0
-void mnodePrintVgroups(SDbObj *pDb, char *oper) {
-  mInfo("db:%s, vgroup link from head, oper:%s", pDb->name, oper);  
+void mnodePrintVgroups(SDbObj *pDb, char *row) {
+  mInfo("db:%s, vgroup link from head, row:%s", pDb->name, row);  
   SVgObj *pVgroup = pDb->pHead;
   while (pVgroup != NULL) {
     mInfo("vgId:%d", pVgroup->vgId);
@@ -807,13 +812,13 @@ static int32_t mnodeSetDbDropping(SDbObj *pDb) {
   if (pDb->status) return TSDB_CODE_SUCCESS;
 
   pDb->status = true;
-  SSdbOper oper = {
+  SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,
-    .table = tsDbSdb,
+    .pTable = tsDbSdb,
     .pObj   = pDb
   };
 
-  int32_t code = sdbUpdateRow(&oper);
+  int32_t code = sdbUpdateRow(&row);
   if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mError("db:%s, failed to set dropping state, reason:%s", pDb->name, tstrerror(code));
   }
@@ -986,8 +991,8 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) {
   SDbObj *pDb = pMsg->pDb;
 
   void *pIter = NULL;
-  while (1) {
   SVgObj *pVgroup = NULL;
+    while (1) {
     pIter = mnodeGetNextVgroup(pIter, &pVgroup);
     if (pVgroup == NULL) break;
     if (pVgroup->pDb == pDb) {
@@ -995,7 +1000,6 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) {
     }
     mnodeDecVgroupRef(pVgroup);
   }
-  sdbFreeIter(pIter);
 
   mDebug("db:%s, all vgroups is altered", pDb->name);
   mLInfo("db:%s, is alterd by %s", pDb->name, mnodeGetUserFromMsg(pMsg));
@@ -1019,15 +1023,15 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SAlterDbMsg *pAlter, void *pMsg) {
   if (memcmp(&newCfg, &pDb->cfg, sizeof(SDbCfg)) != 0) {
     pDb->cfg = newCfg;
     pDb->cfgVersion++;
-    SSdbOper oper = {
+    SSdbRow row = {
       .type    = SDB_OPER_GLOBAL,
-      .table = tsDbSdb,
+      .pTable  = tsDbSdb,
       .pObj    = pDb,
       .pMsg    = pMsg,
-      .writeCb = mnodeAlterDbCb
+      .fpRsp   = mnodeAlterDbCb
     };
 
-    code = sdbUpdateRow(&oper);
+    code = sdbUpdateRow(&row);
     if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
       mError("db:%s, failed to alter, reason:%s", pDb->name, tstrerror(code));
     }
@@ -1071,15 +1075,15 @@ static int32_t mnodeDropDb(SMnodeMsg *pMsg) {
   SDbObj *pDb = pMsg->pDb;
   mInfo("db:%s, drop db from sdb", pDb->name);
 
-  SSdbOper oper = {
+  SSdbRow row = {
     .type    = SDB_OPER_GLOBAL,
-    .table  = tsDbSdb,
+    .pTable  = tsDbSdb,
     .pObj    = pDb,
     .pMsg    = pMsg,
-    .writeCb = mnodeDropDbCb
+    .fpRsp   = mnodeDropDbCb
   };
 
-  int32_t code = sdbDeleteRow(&oper);
+  int32_t code = sdbDeleteRow(&row);
   if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mError("db:%s, failed to drop, reason:%s", pDb->name, tstrerror(code));
   }
@@ -1134,19 +1138,17 @@ void  mnodeDropAllDbs(SAcctObj *pAcct)  {
 
     if (pDb->pAcct == pAcct) {
       mInfo("db:%s, drop db from sdb for acct:%s is dropped", pDb->name, pAcct->user);
-      SSdbOper oper = {
+      SSdbRow row = {
         .type   = SDB_OPER_LOCAL,
-        .table = tsDbSdb,
+        .pTable = tsDbSdb,
         .pObj   = pDb
       };
       
-      sdbDeleteRow(&oper);
+      sdbDeleteRow(&row);
       numOfDbs++;
     }
     mnodeDecDbRef(pDb);
   }
 
-  sdbFreeIter(pIter);
-
   mInfo("acct:%s, all dbs:%d is dropped from sdb", pAcct->user, numOfDbs);
 }

src/mnode/src/mnodeDnode.c

@@ -87,13 +87,13 @@ static char* offlineReason[] = {
   "unknown",
 };
 
-static int32_t mnodeDnodeActionDestroy(SSdbOper *pOper) {
-  tfree(pOper->pObj);
+static int32_t mnodeDnodeActionDestroy(SSdbRow *pRow) {
+  tfree(pRow->pObj);
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeDnodeActionInsert(SSdbOper *pOper) {
-  SDnodeObj *pDnode = pOper->pObj;
+static int32_t mnodeDnodeActionInsert(SSdbRow *pRow) {
+  SDnodeObj *pDnode = pRow->pObj;
   if (pDnode->status != TAOS_DN_STATUS_DROPPING) {
     pDnode->status = TAOS_DN_STATUS_OFFLINE;
     pDnode->lastAccess = tsAccessSquence;
@@ -107,8 +107,8 @@ static int32_t mnodeDnodeActionInsert(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeDnodeActionDelete(SSdbOper *pOper) {
-  SDnodeObj *pDnode = pOper->pObj;
+static int32_t mnodeDnodeActionDelete(SSdbRow *pRow) {
+  SDnodeObj *pDnode = pRow->pObj;
  
 #ifndef _SYNC 
   mnodeDropAllDnodeVgroups(pDnode);
@@ -121,11 +121,11 @@ static int32_t mnodeDnodeActionDelete(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeDnodeActionUpdate(SSdbOper *pOper) {
-  SDnodeObj *pNew = pOper->pObj;
+static int32_t mnodeDnodeActionUpdate(SSdbRow *pRow) {
+  SDnodeObj *pNew = pRow->pObj;
   SDnodeObj *pDnode = mnodeGetDnode(pNew->dnodeId);
   if (pDnode != NULL && pNew != pDnode) {
-    memcpy(pDnode, pNew, pOper->rowSize);
+    memcpy(pDnode, pNew, pRow->rowSize);
     free(pNew);
   }
   mnodeDecDnodeRef(pDnode);
@@ -134,19 +134,19 @@ static int32_t mnodeDnodeActionUpdate(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeDnodeActionEncode(SSdbOper *pOper) {
-  SDnodeObj *pDnode = pOper->pObj;
-  memcpy(pOper->rowData, pDnode, tsDnodeUpdateSize);
-  pOper->rowSize = tsDnodeUpdateSize;
+static int32_t mnodeDnodeActionEncode(SSdbRow *pRow) {
+  SDnodeObj *pDnode = pRow->pObj;
+  memcpy(pRow->rowData, pDnode, tsDnodeUpdateSize);
+  pRow->rowSize = tsDnodeUpdateSize;
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeDnodeActionDecode(SSdbOper *pOper) {
+static int32_t mnodeDnodeActionDecode(SSdbRow *pRow) {
   SDnodeObj *pDnode = (SDnodeObj *) calloc(1, sizeof(SDnodeObj));
   if (pDnode == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
 
-  memcpy(pDnode, pOper->rowData, tsDnodeUpdateSize);
-  pOper->pObj = pDnode;
+  memcpy(pDnode, pRow->rowData, tsDnodeUpdateSize);
+  pRow->pObj = pDnode;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -171,23 +171,23 @@ int32_t mnodeInitDnodes() {
   tsDnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
   pthread_mutex_init(&tsDnodeEpsMutex, NULL);
 
-  SSdbTableDesc tableDesc = {
-    .tableId      = SDB_TABLE_DNODE,
-    .tableName    = "dnodes",
+  SSdbTableDesc desc = {
+    .id           = SDB_TABLE_DNODE,
+    .name         = "dnodes",
     .hashSessions = TSDB_DEFAULT_DNODES_HASH_SIZE,
     .maxRowSize   = tsDnodeUpdateSize,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_AUTO,
-    .insertFp     = mnodeDnodeActionInsert,
-    .deleteFp     = mnodeDnodeActionDelete,
-    .updateFp     = mnodeDnodeActionUpdate,
-    .encodeFp     = mnodeDnodeActionEncode,
-    .decodeFp     = mnodeDnodeActionDecode,
-    .destroyFp    = mnodeDnodeActionDestroy,
-    .restoredFp   = mnodeDnodeActionRestored
+    .fpInsert     = mnodeDnodeActionInsert,
+    .fpDelete     = mnodeDnodeActionDelete,
+    .fpUpdate     = mnodeDnodeActionUpdate,
+    .fpEncode     = mnodeDnodeActionEncode,
+    .fpDecode     = mnodeDnodeActionDecode,
+    .fpDestroy    = mnodeDnodeActionDestroy,
+    .fpRestored   = mnodeDnodeActionRestored
   };
 
-  tsDnodeSdb = sdbOpenTable(&tableDesc);
+  tsDnodeSdb = sdbOpenTable(&desc);
   if (tsDnodeSdb == NULL) {
     mError("failed to init dnodes data");
     return -1;
@@ -206,6 +206,7 @@ int32_t mnodeInitDnodes() {
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, mnodeRetrieveVnodes);
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_DNODE, mnodeGetDnodeMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_DNODE, mnodeRetrieveDnodes);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_DNODE, mnodeCancelGetNextDnode);
  
   mDebug("table:dnodes table is created");
   return 0;
@@ -223,6 +224,10 @@ void *mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode) {
   return sdbFetchRow(tsDnodeSdb, pIter, (void **)pDnode); 
 }
 
+void mnodeCancelGetNextDnode(void *pIter) {
+  sdbFreeIter(tsDnodeSdb, pIter);
+}
+
 int32_t mnodeGetDnodesNum() {
   return sdbGetNumOfRows(tsDnodeSdb);
 }
@@ -241,8 +246,6 @@ int32_t mnodeGetOnlinDnodesCpuCoreNum() {
     mnodeDecDnodeRef(pDnode);
   }
 
-  sdbFreeIter(pIter);
-
   if (cpuCores < 2) cpuCores = 2;
   return cpuCores;
 }
@@ -259,8 +262,6 @@ int32_t mnodeGetOnlineDnodesNum() {
     mnodeDecDnodeRef(pDnode);
   }
 
-  sdbFreeIter(pIter);
-
   return onlineDnodes;
 }
 
@@ -276,13 +277,12 @@ void *mnodeGetDnodeByEp(char *ep) {
     pIter = mnodeGetNextDnode(pIter, &pDnode);
     if (pDnode == NULL) break;
     if (strcmp(ep, pDnode->dnodeEp) == 0) {
-      sdbFreeIter(pIter);
+      mnodeCancelGetNextDnode(pIter);
       return pDnode;
     }
     mnodeDecDnodeRef(pDnode);
   }
 
-  sdbFreeIter(pIter);
 
   return NULL;
 }
@@ -296,13 +296,13 @@ void mnodeDecDnodeRef(SDnodeObj *pDnode) {
 }
 
 void mnodeUpdateDnode(SDnodeObj *pDnode) {
-  SSdbOper oper = {
+  SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,
-    .table = tsDnodeSdb,
+    .pTable = tsDnodeSdb,
     .pObj   = pDnode
   };
 
-  int32_t code = sdbUpdateRow(&oper);
+  int32_t code = sdbUpdateRow(&row);
   if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mError("dnodeId:%d, failed update", pDnode->dnodeId);
   }
@@ -464,7 +464,10 @@ static void mnodeUpdateDnodeEps() {
   while (1) {
     pIter = mnodeGetNextDnode(pIter, &pDnode);
     if (pDnode == NULL) break;
-    if (dnodesNum >= totalDnodes) break;
+    if (dnodesNum >= totalDnodes) {
+      mnodeCancelGetNextDnode(pIter);
+      break;
+    }
 
     SDnodeEp *pEp = &tsDnodeEps->dnodeEps[dnodesNum];
     dnodesNum++;
@@ -474,7 +477,6 @@ static void mnodeUpdateDnodeEps() {
     mnodeDecDnodeRef(pDnode);
   }
 
-  sdbFreeIter(pIter);
   pthread_mutex_unlock(&tsDnodeEpsMutex);
 }
 
@@ -644,15 +646,15 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) {
   tstrncpy(pDnode->dnodeEp, ep, TSDB_EP_LEN);
   taosGetFqdnPortFromEp(ep, pDnode->dnodeFqdn, &pDnode->dnodePort);
 
-  SSdbOper oper = {
+  SSdbRow row = {
     .type    = SDB_OPER_GLOBAL,
-    .table = tsDnodeSdb,
+    .pTable  = tsDnodeSdb,
     .pObj    = pDnode,
     .rowSize = sizeof(SDnodeObj),
     .pMsg    = pMsg
   };
 
-  int32_t code = sdbInsertRow(&oper);
+  int32_t code = sdbInsertRow(&row);
   if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     int dnodeId = pDnode->dnodeId;
     tfree(pDnode);
@@ -665,14 +667,14 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) {
 }
 
 int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg) {
-  SSdbOper oper = {
+  SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,
-    .table = tsDnodeSdb,
+    .pTable = tsDnodeSdb,
     .pObj   = pDnode,
     .pMsg   = pMsg
   };
 
-  int32_t code = sdbDeleteRow(&oper);
+  int32_t code = sdbDeleteRow(&row);
   if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mError("dnode:%d, failed to drop from cluster, result:%s", pDnode->dnodeId, tstrerror(code));
   } else {
@@ -1100,7 +1102,7 @@ static int32_t mnodeGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
     pDnode = mnodeGetDnodeByEp(pShow->payload);
   } else {
     void *pIter = mnodeGetNextDnode(NULL, (SDnodeObj **)&pDnode);
-    sdbFreeIter(pIter);
+    mnodeCancelGetNextDnode(pIter);
   }
 
   if (pDnode != NULL) {
@@ -1141,14 +1143,13 @@ static int32_t mnodeRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, vo
           cols++;
 
           pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-          strcpy(pWrite, mnodeGetMnodeRoleStr(pVgid->role));
+          strcpy(pWrite, syncRole[pVgid->role]);
           cols++;
         }
       }
 
       mnodeDecVgroupRef(pVgroup);
     }
-    sdbFreeIter(pIter);
   } else {
     numOfRows = 0;
   }
@@ -1217,8 +1218,6 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
     mnodeDecDnodeRef(pDnode);
   }
 
-  sdbFreeIter(pIter);
-
   if (pSelDnode == NULL) {
     mError("failed to alloc vnode to vgroup");
     return TSDB_CODE_MND_NO_ENOUGH_DNODES;

src/mnode/src/mnodeInt.c

@@ -48,7 +48,7 @@ void *mnodeCreateMsg(SRpcMsg *pRpcMsg) {
 
 int32_t mnodeInitMsg(SMnodeMsg *pMsg) {
   if (pMsg->pUser != NULL) {
-    mDebug("app:%p:%p, user info already inited", pMsg->rpcMsg.ahandle, pMsg);
+    mTrace("msg:%p, app:%p user info already inited", pMsg, pMsg->rpcMsg.ahandle);
     return TSDB_CODE_SUCCESS;
   }
 

src/mnode/src/mnodeMnode.c

@@ -58,13 +58,13 @@ static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, vo
   #define mnodeMnodeDestroyLock() pthread_mutex_destroy(&tsMnodeLock)
 #endif
 
-static int32_t mnodeMnodeActionDestroy(SSdbOper *pOper) {
-  tfree(pOper->pObj);
+static int32_t mnodeMnodeActionDestroy(SSdbRow *pRow) {
+  tfree(pRow->pObj);
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeMnodeActionInsert(SSdbOper *pOper) {
-  SMnodeObj *pMnode = pOper->pObj;
+static int32_t mnodeMnodeActionInsert(SSdbRow *pRow) {
+  SMnodeObj *pMnode = pRow->pObj;
   SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId);
   if (pDnode == NULL) return TSDB_CODE_MND_DNODE_NOT_EXIST;
 
@@ -76,8 +76,8 @@ static int32_t mnodeMnodeActionInsert(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeMnodeActionDelete(SSdbOper *pOper) {
-  SMnodeObj *pMnode = pOper->pObj;
+static int32_t mnodeMnodeActionDelete(SSdbRow *pRow) {
+  SMnodeObj *pMnode = pRow->pObj;
 
   SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId);
   if (pDnode == NULL) return TSDB_CODE_MND_DNODE_NOT_EXIST;
@@ -88,30 +88,30 @@ static int32_t mnodeMnodeActionDelete(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeMnodeActionUpdate(SSdbOper *pOper) {
-  SMnodeObj *pMnode = pOper->pObj;
+static int32_t mnodeMnodeActionUpdate(SSdbRow *pRow) {
+  SMnodeObj *pMnode = pRow->pObj;
   SMnodeObj *pSaved = mnodeGetMnode(pMnode->mnodeId);
   if (pMnode != pSaved) {
-    memcpy(pSaved, pMnode, pOper->rowSize);
+    memcpy(pSaved, pMnode, pRow->rowSize);
     free(pMnode);
   }
   mnodeDecMnodeRef(pSaved);
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeMnodeActionEncode(SSdbOper *pOper) {
-  SMnodeObj *pMnode = pOper->pObj;
-  memcpy(pOper->rowData, pMnode, tsMnodeUpdateSize);
-  pOper->rowSize = tsMnodeUpdateSize;
+static int32_t mnodeMnodeActionEncode(SSdbRow *pRow) {
+  SMnodeObj *pMnode = pRow->pObj;
+  memcpy(pRow->rowData, pMnode, tsMnodeUpdateSize);
+  pRow->rowSize = tsMnodeUpdateSize;
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeMnodeActionDecode(SSdbOper *pOper) {
+static int32_t mnodeMnodeActionDecode(SSdbRow *pRow) {
   SMnodeObj *pMnode = calloc(1, sizeof(SMnodeObj));
   if (pMnode == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
 
-  memcpy(pMnode, pOper->rowData, tsMnodeUpdateSize);
-  pOper->pObj = pMnode;
+  memcpy(pMnode, pRow->rowData, tsMnodeUpdateSize);
+  pRow->pObj = pMnode;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -123,7 +123,7 @@ static int32_t mnodeMnodeActionRestored() {
       pMnode->role = TAOS_SYNC_ROLE_MASTER;
       mnodeDecMnodeRef(pMnode);
     }
-    sdbFreeIter(pIter);
+    mnodeCancelGetNextMnode(pIter);
   }
 
   mnodeUpdateMnodeEpSet();
@@ -137,23 +137,23 @@ int32_t mnodeInitMnodes() {
   SMnodeObj tObj;
   tsMnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
 
-  SSdbTableDesc tableDesc = {
-    .tableId      = SDB_TABLE_MNODE,
-    .tableName    = "mnodes",
+  SSdbTableDesc desc = {
+    .id           = SDB_TABLE_MNODE,
+    .name         = "mnodes",
     .hashSessions = TSDB_DEFAULT_MNODES_HASH_SIZE,
     .maxRowSize   = tsMnodeUpdateSize,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_INT,
-    .insertFp     = mnodeMnodeActionInsert,
-    .deleteFp     = mnodeMnodeActionDelete,
-    .updateFp     = mnodeMnodeActionUpdate,
-    .encodeFp     = mnodeMnodeActionEncode,
-    .decodeFp     = mnodeMnodeActionDecode,
-    .destroyFp    = mnodeMnodeActionDestroy,
-    .restoredFp   = mnodeMnodeActionRestored
+    .fpInsert     = mnodeMnodeActionInsert,
+    .fpDelete     = mnodeMnodeActionDelete,
+    .fpUpdate     = mnodeMnodeActionUpdate,
+    .fpEncode     = mnodeMnodeActionEncode,
+    .fpDecode     = mnodeMnodeActionDecode,
+    .fpDestroy    = mnodeMnodeActionDestroy,
+    .fpRestored   = mnodeMnodeActionRestored
   };
 
-  tsMnodeSdb = sdbOpenTable(&tableDesc);
+  tsMnodeSdb = sdbOpenTable(&desc);
   if (tsMnodeSdb == NULL) {
     mError("failed to init mnodes data");
     return -1;
@@ -161,6 +161,7 @@ int32_t mnodeInitMnodes() {
 
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mnodeGetMnodeMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mnodeRetrieveMnodes);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_MNODE, mnodeCancelGetNextMnode);
 
   mDebug("table:mnodes table is created");
   return TSDB_CODE_SUCCESS;
@@ -192,8 +193,8 @@ void *mnodeGetNextMnode(void *pIter, SMnodeObj **pMnode) {
   return sdbFetchRow(tsMnodeSdb, pIter, (void **)pMnode); 
 }
 
-char *mnodeGetMnodeRoleStr(int32_t role) {
-  return syncRole[role];
+void mnodeCancelGetNextMnode(void *pIter) {
+  sdbFreeIter(tsMnodeSdb, pIter);
 }
 
 void mnodeUpdateMnodeEpSet() {
@@ -243,8 +244,6 @@ void mnodeUpdateMnodeEpSet() {
   tsMnodeEpSetForShell.numOfEps = index;
   tsMnodeEpSetForPeer.numOfEps = index;
 
-  sdbFreeIter(pIter);
-
   mnodeMnodeUnLock();
 }
 
@@ -329,11 +328,11 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) {
   pMnode->mnodeId = dnodeId;
   pMnode->createdTime = taosGetTimestampMs();
 
-  SSdbOper oper = {
+  SSdbRow row = {
     .type    = SDB_OPER_GLOBAL,
-    .table = tsMnodeSdb,
+    .pTable  = tsMnodeSdb,
     .pObj    = pMnode,
-    .writeCb = mnodeCreateMnodeCb
+    .fpRsp   = mnodeCreateMnodeCb
   };
 
   int32_t code = TSDB_CODE_SUCCESS;
@@ -346,7 +345,7 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) {
     return;
   }
 
-  code = sdbInsertRow(&oper);
+  code = sdbInsertRow(&row);
   if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mError("dnode:%d, failed to create mnode, ep:%s reason:%s", dnodeId, dnodeEp, tstrerror(code));
     tfree(pMnode);
@@ -356,8 +355,8 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) {
 void mnodeDropMnodeLocal(int32_t dnodeId) {
   SMnodeObj *pMnode = mnodeGetMnode(dnodeId);
   if (pMnode != NULL) {
-    SSdbOper oper = {.type = SDB_OPER_LOCAL, .table = tsMnodeSdb, .pObj = pMnode};
-    sdbDeleteRow(&oper);
+    SSdbRow row = {.type = SDB_OPER_LOCAL, .pTable = tsMnodeSdb, .pObj = pMnode};
+    sdbDeleteRow(&row);
     mnodeDecMnodeRef(pMnode);
   }
 
@@ -371,13 +370,13 @@ int32_t mnodeDropMnode(int32_t dnodeId) {
     return TSDB_CODE_MND_DNODE_NOT_EXIST;
   }
   
-  SSdbOper oper = {
+  SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,
-    .table = tsMnodeSdb,
+    .pTable = tsMnodeSdb,
     .pObj   = pMnode
   };
 
-  int32_t code = sdbDeleteRow(&oper);
+  int32_t code = sdbDeleteRow(&row);
 
   sdbDecRef(tsMnodeSdb, pMnode);
 
@@ -469,7 +468,7 @@ static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, vo
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    char* roles = mnodeGetMnodeRoleStr(pMnode->role);
+    char* roles = syncRole[pMnode->role];
     STR_WITH_MAXSIZE_TO_VARSTR(pWrite, roles, pShow->bytes[cols]);
     cols++;
 

src/mnode/src/mnodePeer.c

@@ -47,7 +47,7 @@ void mnodeAddPeerRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) {
 
 int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) {
   if (pMsg->rpcMsg.pCont == NULL) {
-    mError("%p, msg:%s in mpeer queue, content is null", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]);
+    mError("msg:%p, ahandle:%p type:%s in mpeer queue, content is null", pMsg, pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]);
     return TSDB_CODE_MND_INVALID_MSG_LEN;
   }
 
@@ -58,8 +58,8 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) {
     rpcRsp->rsp = epSet;
     rpcRsp->len = sizeof(SRpcEpSet);
 
-    mDebug("%p, msg:%s in mpeer queue will be redirected, numOfEps:%d inUse:%d", pMsg->rpcMsg.ahandle,
-           taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse);
+    mDebug("msg:%p, ahandle:%p type:%s in mpeer queue will be redirected, numOfEps:%d inUse:%d", pMsg,
+           pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse);
     for (int32_t i = 0; i < epSet->numOfEps; ++i) {
       if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort + TSDB_PORT_DNODEDNODE) {
         epSet->inUse = (i + 1) % epSet->numOfEps;
@@ -73,7 +73,8 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) {
   }
 
   if (tsMnodeProcessPeerMsgFp[pMsg->rpcMsg.msgType] == NULL) {
-    mError("%p, msg:%s in mpeer queue, not processed", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]);
+    mError("msg:%p, ahandle:%p type:%s in mpeer queue, not processed", pMsg, pMsg->rpcMsg.ahandle,
+           taosMsg[pMsg->rpcMsg.msgType]);
     return TSDB_CODE_MND_MSG_NOT_PROCESSED;
   }
 
@@ -82,13 +83,14 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) {
 
 void mnodeProcessPeerRsp(SRpcMsg *pMsg) {
   if (!sdbIsMaster()) {
-    mError("%p, msg:%s is not processed for it is not master", pMsg->ahandle, taosMsg[pMsg->msgType]);
+    mError("msg:%p, ahandle:%p type:%s  is not processed for it is not master", pMsg, pMsg->ahandle,
+           taosMsg[pMsg->msgType]);
     return;
   }
 
   if (tsMnodeProcessPeerRspFp[pMsg->msgType]) {
     (*tsMnodeProcessPeerRspFp[pMsg->msgType])(pMsg);
   } else {
-    mError("%p, msg:%s is not processed", pMsg->ahandle, taosMsg[pMsg->msgType]);
+    mError("msg:%p, ahandle:%p type:%s is not processed", pMsg, pMsg->ahandle, taosMsg[pMsg->msgType]);
   }
 }

src/mnode/src/mnodeProfile.c

@@ -42,6 +42,7 @@ static int32_t mnodeGetQueryMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
 static int32_t mnodeRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 static int32_t mnodeGetConnsMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mnodeRetrieveConns(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+static void    mnodeCancelGetNextConn(void *pIter);
 static int32_t mnodeGetStreamMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mnodeRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, void *pConn);
 static void    mnodeFreeConn(void *data);
@@ -52,10 +53,13 @@ static int32_t mnodeProcessKillConnectionMsg(SMnodeMsg *pMsg);
 int32_t mnodeInitProfile() {
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_QUERIES, mnodeGetQueryMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_QUERIES, mnodeRetrieveQueries);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_QUERIES, mnodeCancelGetNextConn);
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_CONNS, mnodeGetConnsMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_CONNS, mnodeRetrieveConns);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_CONNS, mnodeCancelGetNextConn);
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_STREAMS, mnodeGetStreamMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_STREAMS, mnodeRetrieveStreams);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_STREAMS, mnodeCancelGetNextConn);
 
   mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_QUERY, mnodeProcessKillQueryMsg);
   mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_STREAM, mnodeProcessKillStreamMsg);
@@ -137,21 +141,15 @@ static void mnodeFreeConn(void *data) {
   mDebug("connId:%d, is destroyed", pConn->connId);
 }
 
-static void *mnodeGetNextConn(SHashMutableIterator *pIter, SConnObj **pConn) {
+static void *mnodeGetNextConn(void *pIter, SConnObj **pConn) {
   *pConn = NULL;
 
-  if (pIter == NULL) {
-    pIter = taosHashCreateIter(tsMnodeConnCache->pHashTable);
-  }
-
-  if (!taosHashIterNext(pIter)) {
-    taosHashDestroyIter(pIter);
-    return NULL;
-  }
+  pIter = taosHashIterate(tsMnodeConnCache->pHashTable, pIter);
+  if (pIter == NULL) return NULL;
 
-  SCacheDataNode **pNode = taosHashIterGet(pIter);
+  SCacheDataNode **pNode = pIter;
   if (pNode == NULL || *pNode == NULL) {
-    taosHashDestroyIter(pIter);
+    taosHashCancelIterate(tsMnodeConnCache->pHashTable, pIter);
     return NULL;
   }
 
@@ -159,6 +157,10 @@ static void *mnodeGetNextConn(SHashMutableIterator *pIter, SConnObj **pConn) {
   return pIter;
 }
 
+static void mnodeCancelGetNextConn(void *pIter) {
+  taosHashCancelIterate(tsMnodeConnCache->pHashTable, pIter);
+}
+
 static int32_t mnodeGetConnsMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   SUserObj *pUser = mnodeGetUserFromConn(pConn);
   if (pUser == NULL) return 0;

src/mnode/src/mnodeRead.c

@@ -43,7 +43,7 @@ void mnodeAddReadMsgHandle(uint8_t msgType, int32_t (*fp)(SMnodeMsg *pMsg)) {
 
 int32_t mnodeProcessRead(SMnodeMsg *pMsg) {
   if (pMsg->rpcMsg.pCont == NULL) {
-    mError("%p, msg:%s in mread queue, content is null", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]);
+    mError("msg:%p, app:%p type:%s in mread queue, content is null", pMsg, pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]);
     return TSDB_CODE_MND_INVALID_MSG_LEN;
   }
 
@@ -52,7 +52,7 @@ int32_t mnodeProcessRead(SMnodeMsg *pMsg) {
     SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet));
     mnodeGetMnodeEpSetForShell(epSet);
 
-    mDebug("%p, msg:%s in mread queue will be redirected, numOfEps:%d inUse:%d", pMsg->rpcMsg.ahandle,
+    mDebug("msg:%p, app:%p type:%s in mread queue will be redirected, numOfEps:%d inUse:%d", pMsg, pMsg->rpcMsg.ahandle,
            taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse);
     for (int32_t i = 0; i < epSet->numOfEps; ++i) {
       if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) {
@@ -70,13 +70,15 @@ int32_t mnodeProcessRead(SMnodeMsg *pMsg) {
   }
 
   if (tsMnodeProcessReadMsgFp[pMsg->rpcMsg.msgType] == NULL) {
-    mError("%p, msg:%s in mread queue, not processed", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]);
+    mError("msg:%p, app:%p type:%s in mread queue, not processed", pMsg, pMsg->rpcMsg.ahandle,
+           taosMsg[pMsg->rpcMsg.msgType]);
     return TSDB_CODE_MND_MSG_NOT_PROCESSED;
   }
 
   int32_t code = mnodeInitMsg(pMsg);
   if (code != TSDB_CODE_SUCCESS) {
-    mError("%p, msg:%s in mread queue, not processed reason:%s", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], tstrerror(code));
+    mError("msg:%p, app:%p type:%s in mread queue, not processed reason:%s", pMsg, pMsg->rpcMsg.ahandle,
+           taosMsg[pMsg->rpcMsg.msgType], tstrerror(code));
     return code;
   }
 

src/mnode/src/mnodeSdb.c

@@ -34,23 +34,30 @@
 #include "mnodeSdb.h"
 
 #define SDB_TABLE_LEN 12
-#define SDB_SYNC_HACK 16
+#define MAX_QUEUED_MSG_NUM 10000
 
 typedef enum {
-  SDB_ACTION_INSERT,
-  SDB_ACTION_DELETE,
-  SDB_ACTION_UPDATE
+  SDB_ACTION_INSERT = 0,
+  SDB_ACTION_DELETE = 1,
+  SDB_ACTION_UPDATE = 2
 } ESdbAction;
 
 typedef enum {
-  SDB_STATUS_OFFLINE,
-  SDB_STATUS_SERVING,
-  SDB_STATUS_CLOSING
+  SDB_STATUS_OFFLINE = 0,
+  SDB_STATUS_SERVING = 1,
+  SDB_STATUS_CLOSING = 2
 } ESdbStatus;
 
-typedef struct _SSdbTable {
-  char      tableName[SDB_TABLE_LEN];
-  ESdbTable tableId;
+char *actStr[] = {
+  "insert",
+  "delete",
+  "update",
+  "invalid"
+};
+
+typedef struct SSdbTable {
+  char      name[SDB_TABLE_LEN];
+  ESdbTable id;
   ESdbKey   keyType;
   int32_t   hashSessions;
   int32_t   maxRowSize;
@@ -58,13 +65,13 @@ typedef struct _SSdbTable {
   int32_t   autoIndex;
   int64_t   numOfRows;
   void *    iHandle;
-  int32_t (*insertFp)(SSdbOper *pDesc);
-  int32_t (*deleteFp)(SSdbOper *pOper);
-  int32_t (*updateFp)(SSdbOper *pOper);
-  int32_t (*decodeFp)(SSdbOper *pOper);
-  int32_t (*encodeFp)(SSdbOper *pOper);
-  int32_t (*destroyFp)(SSdbOper *pOper);
-  int32_t (*restoredFp)();
+  int32_t (*fpInsert)(SSdbRow *pRow);
+  int32_t (*fpDelete)(SSdbRow *pRow);
+  int32_t (*fpUpdate)(SSdbRow *pRow);
+  int32_t (*fpDecode)(SSdbRow *pRow);
+  int32_t (*fpEncode)(SSdbRow *pRow);
+  int32_t (*fpDestroy)(SSdbRow *pRow);
+  int32_t (*fpRestored)();
   pthread_mutex_t mutex;
 } SSdbTable;
 
@@ -75,60 +82,61 @@ typedef struct {
   int64_t    sync;
   void *     wal;
   SSyncCfg   cfg;
+  int32_t    queuedMsg;
   int32_t    numOfTables;
   SSdbTable *tableList[SDB_TABLE_MAX];
   pthread_mutex_t mutex;
-} SSdbObject;
+} SSdbMgmt;
 
 typedef struct {
   pthread_t thread;
   int32_t   workerId;
-} SSdbWriteWorker;
+} SSdbWorker;
 
 typedef struct {
   int32_t num;
-  SSdbWriteWorker *writeWorker;
-} SSdbWriteWorkerPool;
+  SSdbWorker *worker;
+} SSdbWorkerPool;
 
 extern void *     tsMnodeTmr;
-static void *     tsUpdateSyncTmr;
-static SSdbObject tsSdbObj = {0};
-static taos_qset  tsSdbWriteQset;
-static taos_qall  tsSdbWriteQall;
-static taos_queue tsSdbWriteQueue;
-static SSdbWriteWorkerPool tsSdbPool;
-
-static int32_t sdbWrite(void *param, void *data, int32_t type, void *pMsg);
-static int32_t sdbWriteToQueue(void *param, void *data, int32_t type, void *pMsg);
-static void *  sdbWorkerFp(void *param);
-static int32_t sdbInitWriteWorker();
-static void    sdbCleanupWriteWorker();
-static int32_t sdbAllocWriteQueue();
-static void    sdbFreeWritequeue();
-static int32_t sdbUpdateRowImp(SSdbOper *pOper);
-static int32_t sdbDeleteRowImp(SSdbOper *pOper);
-static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper);
-static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper);
-static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper);
-
-int32_t sdbGetId(void *handle) {
-  return ((SSdbTable *)handle)->autoIndex;
+static void *     tsSdbTmr;
+static SSdbMgmt   tsSdbMgmt = {0};
+static taos_qset  tsSdbWQset;
+static taos_qall  tsSdbWQall;
+static taos_queue tsSdbWQueue;
+static SSdbWorkerPool tsSdbPool;
+
+static int32_t sdbProcessWrite(void *pRow, void *pHead, int32_t qtype, void *unused);
+static int32_t sdbWriteFwdToQueue(int32_t vgId, void *pHead, int32_t qtype, void *rparam);
+static int32_t sdbWriteRowToQueue(SSdbRow *pRow, int32_t action);
+static void    sdbFreeFromQueue(SSdbRow *pRow);
+static void *  sdbWorkerFp(void *pWorker);
+static int32_t sdbInitWorker();
+static void    sdbCleanupWorker();
+static int32_t sdbAllocQueue();
+static void    sdbFreeQueue();
+static int32_t sdbInsertHash(SSdbTable *pTable, SSdbRow *pRow);
+static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbRow *pRow);
+static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow);
+
+int32_t sdbGetId(void *pTable) {
+  return ((SSdbTable *)pTable)->autoIndex;
 }
 
-int64_t sdbGetNumOfRows(void *handle) {
-  return ((SSdbTable *)handle)->numOfRows;
+int64_t sdbGetNumOfRows(void *pTable) {
+  return ((SSdbTable *)pTable)->numOfRows;
 }
 
 uint64_t sdbGetVersion() {
-  return tsSdbObj.version;
+  return tsSdbMgmt.version;
 }
 
 bool sdbIsMaster() { 
-  return tsSdbObj.role == TAOS_SYNC_ROLE_MASTER; 
+  return tsSdbMgmt.role == TAOS_SYNC_ROLE_MASTER; 
 }
 
 bool sdbIsServing() {
-  return tsSdbObj.status == SDB_STATUS_SERVING; 
+  return tsSdbMgmt.status == SDB_STATUS_SERVING; 
 }
 
 static void *sdbGetObjKey(SSdbTable *pTable, void *key) {
@@ -139,18 +147,6 @@ static void *sdbGetObjKey(SSdbTable *pTable, void *key) {
   return key;
 }
 
-static char *sdbGetActionStr(int32_t action) {
-  switch (action) {
-    case SDB_ACTION_INSERT:
-      return "insert";
-    case SDB_ACTION_DELETE:
-      return "delete";
-    case SDB_ACTION_UPDATE:
-      return "update";
-  }
-  return "invalid";
-}
-
 static char *sdbGetKeyStr(SSdbTable *pTable, void *key) {
   static char str[16];
   switch (pTable->keyType) {
@@ -166,28 +162,28 @@ static char *sdbGetKeyStr(SSdbTable *pTable, void *key) {
   }
 }
 
-static char *sdbGetKeyStrFromObj(SSdbTable *pTable, void *key) {
+static char *sdbGetRowStr(SSdbTable *pTable, void *key) {
   return sdbGetKeyStr(pTable, sdbGetObjKey(pTable, key));
 }
 
 static void *sdbGetTableFromId(int32_t tableId) {
-  return tsSdbObj.tableList[tableId];
+  return tsSdbMgmt.tableList[tableId];
 }
 
 static int32_t sdbInitWal() {
   SWalCfg walCfg = {.vgId = 1, .walLevel = TAOS_WAL_FSYNC, .keep = TAOS_WAL_KEEP, .fsyncPeriod = 0};
-  char temp[TSDB_FILENAME_LEN];
+  char    temp[TSDB_FILENAME_LEN] = {0};
   sprintf(temp, "%s/wal", tsMnodeDir);
-  tsSdbObj.wal = walOpen(temp, &walCfg);
-  if (tsSdbObj.wal == NULL) {
-    sdbError("failed to open sdb wal in %s", tsMnodeDir);
+  tsSdbMgmt.wal = walOpen(temp, &walCfg);
+  if (tsSdbMgmt.wal == NULL) {
+    sdbError("vgId:1, failed to open wal in %s", tsMnodeDir);
     return -1;
   }
 
-  sdbInfo("open sdb wal for restore");
-  int code = walRestore(tsSdbObj.wal, NULL, sdbWrite);
+  sdbInfo("vgId:1, open wal for restore");
+  int32_t code = walRestore(tsSdbMgmt.wal, NULL, sdbProcessWrite);
   if (code != TSDB_CODE_SUCCESS) {
-    sdbError("failed to open wal for restore, reason:%s", tstrerror(code));
+    sdbError("vgId:1, failed to open wal for restore since %s", tstrerror(code));
     return -1;
   }
   return 0;
@@ -199,31 +195,31 @@ static void sdbRestoreTables() {
   for (int32_t tableId = 0; tableId < SDB_TABLE_MAX; ++tableId) {
     SSdbTable *pTable = sdbGetTableFromId(tableId);
     if (pTable == NULL) continue;
-    if (pTable->restoredFp) {
-      (*pTable->restoredFp)();
+    if (pTable->fpRestored) {
+      (*pTable->fpRestored)();
     }
 
     totalRows += pTable->numOfRows;
     numOfTables++;
-    sdbDebug("table:%s, is restored, numOfRows:%" PRId64, pTable->tableName, pTable->numOfRows);
+    sdbDebug("vgId:1, sdb:%s is restored, rows:%" PRId64, pTable->name, pTable->numOfRows);
   }
 
-  sdbInfo("sdb is restored, ver:%" PRId64 " totalRows:%d numOfTables:%d", tsSdbObj.version, totalRows, numOfTables);
+  sdbInfo("vgId:1, sdb is restored, mver:%" PRIu64 " rows:%d tables:%d", tsSdbMgmt.version, totalRows, numOfTables);
 }
 
 void sdbUpdateMnodeRoles() {
-  if (tsSdbObj.sync <= 0) return;
+  if (tsSdbMgmt.sync <= 0) return;
 
   SNodesRole roles = {0};
-  syncGetNodesRole(tsSdbObj.sync, &roles);
+  if (syncGetNodesRole(tsSdbMgmt.sync, &roles) != 0) return;
 
-  sdbInfo("update mnodes sync roles, total:%d", tsSdbObj.cfg.replica);
-  for (int32_t i = 0; i < tsSdbObj.cfg.replica; ++i) {
+  sdbInfo("vgId:1, update mnodes role, replica:%d", tsSdbMgmt.cfg.replica);
+  for (int32_t i = 0; i < tsSdbMgmt.cfg.replica; ++i) {
     SMnodeObj *pMnode = mnodeGetMnode(roles.nodeId[i]);
     if (pMnode != NULL) {
       pMnode->role = roles.role[i];
-      sdbInfo("mnode:%d, role:%s", pMnode->mnodeId, mnodeGetMnodeRoleStr(pMnode->role));
-      if (pMnode->mnodeId == dnodeGetDnodeId()) tsSdbObj.role = pMnode->role;
+      sdbInfo("vgId:1, mnode:%d, role:%s", pMnode->mnodeId, syncRole[pMnode->role]);
+      if (pMnode->mnodeId == dnodeGetDnodeId()) tsSdbMgmt.role = pMnode->role;
       mnodeDecMnodeRef(pMnode);
     }
   }
@@ -232,93 +228,80 @@ void sdbUpdateMnodeRoles() {
   mnodeUpdateMnodeEpSet();
 }
 
-static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) {
+static uint32_t sdbGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) {
   sdbUpdateMnodeRoles();
   return 0;
 }
 
-static int32_t sdbGetWalInfo(void *ahandle, char *fileName, int64_t *fileId) {
-  return walGetWalFile(tsSdbObj.wal, fileName, fileId);
+static int32_t sdbGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId) {
+  return walGetWalFile(tsSdbMgmt.wal, fileName, fileId);
 }
 
-static void sdbNotifyRole(void *ahandle, int8_t role) {
-  sdbInfo("mnode role changed from %s to %s", mnodeGetMnodeRoleStr(tsSdbObj.role), mnodeGetMnodeRoleStr(role));
+static void sdbNotifyRole(int32_t vgId, int8_t role) {
+  sdbInfo("vgId:1, mnode role changed from %s to %s", syncRole[tsSdbMgmt.role], syncRole[role]);
 
-  if (role == TAOS_SYNC_ROLE_MASTER && tsSdbObj.role != TAOS_SYNC_ROLE_MASTER) {
+  if (role == TAOS_SYNC_ROLE_MASTER && tsSdbMgmt.role != TAOS_SYNC_ROLE_MASTER) {
     balanceReset();
   }
-  tsSdbObj.role = role;
+  tsSdbMgmt.role = role;
 
   sdbUpdateMnodeRoles();
 }
 
-FORCE_INLINE
-static void sdbConfirmForward(void *ahandle, void *param, int32_t code) {
-  assert(param);
-  SSdbOper * pOper = param;
-  SMnodeMsg *pMsg = pOper->pMsg;
-  if (code <= 0) pOper->retCode = code;
-
-  int32_t processedCount = atomic_add_fetch_32(&pOper->processedCount, 1);
-  if (processedCount <= 1) {
-    if (pMsg != NULL) {
-      sdbDebug("app:%p:%p, waiting for confirm this operation, count:%d result:%s", pMsg->rpcMsg.ahandle, pMsg,
-               processedCount, tstrerror(code));
-    }
-    return;
-  }
+// failed to forward, need revert insert
+static void sdbHandleFailedConfirm(SSdbRow *pRow) {
+  SWalHead *pHead = pRow->pHead;
+  int32_t   action = pHead->msgType % 10;
 
-  if (pMsg != NULL) {
-    sdbDebug("app:%p:%p, is confirmed and will do callback func, result:%s", pMsg->rpcMsg.ahandle, pMsg,
-             tstrerror(code));
-  }
+  sdbError("vgId:1, row:%p:%s hver:%" PRIu64 " action:%s, failed to foward since %s", pRow->pObj,
+           sdbGetKeyStr(pRow->pTable, pHead->cont), pHead->version, actStr[action], tstrerror(pRow->code));
 
-  // failed to forward, need revert insert
-  if (pOper->retCode != TSDB_CODE_SUCCESS) {
-    SWalHead *pHead = (void *)pOper + sizeof(SSdbOper) + SDB_SYNC_HACK;
-    int32_t   action = pHead->msgType % 10;
-    sdbError("table:%s record:%p:%s ver:%" PRIu64 ", action:%d failed to foward reason:%s",
-             ((SSdbTable *)pOper->table)->tableName, pOper->pObj, sdbGetKeyStr(pOper->table, pHead->cont),
-             pHead->version, action, tstrerror(pOper->retCode));
-    if (action == SDB_ACTION_INSERT) {
   // It's better to create a table in two stages, create it first and then set it success
-      //sdbDeleteHash(pOper->table, pOper);
-      SSdbOper oper = {
-        .type  = SDB_OPER_GLOBAL,
-        .table = pOper->table,
-        .pObj  = pOper->pObj
-      };
-      sdbDeleteRow(&oper);
-    }
+  if (action == SDB_ACTION_INSERT) {
+    SSdbRow row = {.type = SDB_OPER_GLOBAL, .pTable = pRow->pTable, .pObj = pRow->pObj};
+    sdbDeleteRow(&row);
   }
+}
 
-  if (pOper->writeCb != NULL) {
-    pOper->retCode = (*pOper->writeCb)(pMsg, pOper->retCode);
+FORCE_INLINE
+static void sdbConfirmForward(int32_t vgId, void *wparam, int32_t code) {
+  if (wparam == NULL) return;
+  SSdbRow *pRow = wparam;
+  SMnodeMsg * pMsg = pRow->pMsg;
+
+  if (code <= 0) pRow->code = code;
+  int32_t count = atomic_add_fetch_32(&pRow->processedCount, 1);
+  if (count <= 1) {
+    if (pMsg != NULL) sdbTrace("vgId:1, msg:%p waiting for confirm, count:%d code:%x", pMsg, count, code);
+    return;
+  } else {
+    if (pMsg != NULL) sdbTrace("vgId:1, msg:%p is confirmed, code:%x", pMsg, code);
   }
-  dnodeSendRpcMWriteRsp(pMsg, pOper->retCode);
 
-  // if ahandle, means this func is called by sdb write
-  if (ahandle == NULL) {
-    sdbDecRef(pOper->table, pOper->pObj);
+  if (pRow->code != TSDB_CODE_SUCCESS) sdbHandleFailedConfirm(pRow);
+
+  if (pRow->fpRsp != NULL) {
+    pRow->code = (*pRow->fpRsp)(pMsg, pRow->code);
   }
 
-  taosFreeQitem(pOper);
+  dnodeSendRpcMWriteRsp(pMsg, pRow->code);
+  sdbFreeFromQueue(pRow);
 }
 
 static void sdbUpdateSyncTmrFp(void *param, void *tmrId) { sdbUpdateSync(NULL); }
 
 void sdbUpdateAsync() {
-  taosTmrReset(sdbUpdateSyncTmrFp, 200, NULL, tsMnodeTmr, &tsUpdateSyncTmr);
+  taosTmrReset(sdbUpdateSyncTmrFp, 200, NULL, tsMnodeTmr, &tsSdbTmr);
 }
 
 void sdbUpdateSync(void *pMnodes) {
   SMnodeInfos *mnodes = pMnodes;
   if (!mnodeIsRunning()) {
-    mDebug("mnode not start yet, update sync config later");
+    mDebug("vgId:1, mnode not start yet, update sync config later");
     return;
   }
 
-  mDebug("update sync config in sync module, mnodes:%p", pMnodes);
+  mDebug("vgId:1, update sync config in sync module, mnodes:%p", pMnodes);
 
   SSyncCfg syncCfg = {0};
   int32_t  index = 0;
@@ -342,9 +325,8 @@ void sdbUpdateSync(void *pMnodes) {
       mnodeDecDnodeRef(pDnode);
       mnodeDecMnodeRef(pMnode);
     }
-    sdbFreeIter(pIter);
     syncCfg.replica = index;
-    mDebug("mnodes info not input, use infos in sdb, numOfMnodes:%d", syncCfg.replica);
+    mDebug("vgId:1, mnodes info not input, use infos in sdb, numOfMnodes:%d", syncCfg.replica);
   } else {
     for (index = 0; index < mnodes->mnodeNum; ++index) {
       SMnodeInfo *node = &mnodes->mnodeInfos[index];
@@ -353,7 +335,7 @@ void sdbUpdateSync(void *pMnodes) {
       syncCfg.nodeInfo[index].nodePort += TSDB_PORT_SYNC;
     }
     syncCfg.replica = index;
-    mDebug("mnodes info input, numOfMnodes:%d", syncCfg.replica);
+    mDebug("vgId:1, mnodes info input, numOfMnodes:%d", syncCfg.replica);
   }
 
   syncCfg.quorum = (syncCfg.replica == 1) ? 1 : 2;
@@ -367,18 +349,19 @@ void sdbUpdateSync(void *pMnodes) {
   }
 
   if (!hasThisDnode) {
-    sdbDebug("update sync config, this dnode not exist");
+    sdbDebug("vgId:1, update sync config, this dnode not exist");
     return;
   }
 
-  if (memcmp(&syncCfg, &tsSdbObj.cfg, sizeof(SSyncCfg)) == 0) {
-    sdbDebug("update sync config, info not changed");
+  if (memcmp(&syncCfg, &tsSdbMgmt.cfg, sizeof(SSyncCfg)) == 0) {
+    sdbDebug("vgId:1, update sync config, info not changed");
     return;
   }
 
-  sdbInfo("work as mnode, replica:%d", syncCfg.replica);
+  sdbInfo("vgId:1, work as mnode, replica:%d", syncCfg.replica);
   for (int32_t i = 0; i < syncCfg.replica; ++i) {
-    sdbInfo("mnode:%d, %s:%d", syncCfg.nodeInfo[i].nodeId, syncCfg.nodeInfo[i].nodeFqdn, syncCfg.nodeInfo[i].nodePort);
+    sdbInfo("vgId:1, mnode:%d, %s:%d", syncCfg.nodeInfo[i].nodeId, syncCfg.nodeInfo[i].nodeFqdn,
+            syncCfg.nodeInfo[i].nodePort);
   }
 
   SSyncInfo syncInfo = {0};
@@ -386,26 +369,26 @@ void sdbUpdateSync(void *pMnodes) {
   syncInfo.version = sdbGetVersion();
   syncInfo.syncCfg = syncCfg;
   sprintf(syncInfo.path, "%s", tsMnodeDir);
-  syncInfo.ahandle = NULL;
   syncInfo.getWalInfo = sdbGetWalInfo;
   syncInfo.getFileInfo = sdbGetFileInfo;
-  syncInfo.writeToCache = sdbWriteToQueue;
+  syncInfo.writeToCache = sdbWriteFwdToQueue;
   syncInfo.confirmForward = sdbConfirmForward;
   syncInfo.notifyRole = sdbNotifyRole;
-  tsSdbObj.cfg = syncCfg;
+  tsSdbMgmt.cfg = syncCfg;
 
-  if (tsSdbObj.sync) {
-    syncReconfig(tsSdbObj.sync, &syncCfg);
+  if (tsSdbMgmt.sync) {
+    syncReconfig(tsSdbMgmt.sync, &syncCfg);
   } else {
-    tsSdbObj.sync = syncStart(&syncInfo);
+    tsSdbMgmt.sync = syncStart(&syncInfo);
   }
+
   sdbUpdateMnodeRoles();
 }
 
 int32_t sdbInit() {
-  pthread_mutex_init(&tsSdbObj.mutex, NULL);
+  pthread_mutex_init(&tsSdbMgmt.mutex, NULL);
 
-  if (sdbInitWriteWorker() != 0) {
+  if (sdbInitWorker() != 0) {
     return -1;
   }
 
@@ -416,56 +399,56 @@ int32_t sdbInit() {
   sdbRestoreTables();
 
   if (mnodeGetMnodesNum() == 1) {
-    tsSdbObj.role = TAOS_SYNC_ROLE_MASTER;
+    tsSdbMgmt.role = TAOS_SYNC_ROLE_MASTER;
   }
 
-  tsSdbObj.status = SDB_STATUS_SERVING;
+  tsSdbMgmt.status = SDB_STATUS_SERVING;
   return TSDB_CODE_SUCCESS;
 }
 
 void sdbCleanUp() {
-  if (tsSdbObj.status != SDB_STATUS_SERVING) return;
+  if (tsSdbMgmt.status != SDB_STATUS_SERVING) return;
 
-  tsSdbObj.status = SDB_STATUS_CLOSING;
+  tsSdbMgmt.status = SDB_STATUS_CLOSING;
 
-  sdbCleanupWriteWorker();
-  sdbDebug("sdb will be closed, ver:%" PRId64, tsSdbObj.version);
+  sdbCleanupWorker();
+  sdbDebug("vgId:1, sdb will be closed, mver:%" PRIu64, tsSdbMgmt.version);
 
-  if (tsSdbObj.sync) {
-    syncStop(tsSdbObj.sync);
-    tsSdbObj.sync = -1;
+  if (tsSdbMgmt.sync) {
+    syncStop(tsSdbMgmt.sync);
+    tsSdbMgmt.sync = -1;
   }
 
-  if (tsSdbObj.wal) {
-    walClose(tsSdbObj.wal);
-    tsSdbObj.wal = NULL;
+  if (tsSdbMgmt.wal) {
+    walClose(tsSdbMgmt.wal);
+    tsSdbMgmt.wal = NULL;
   }
   
-  pthread_mutex_destroy(&tsSdbObj.mutex);
+  pthread_mutex_destroy(&tsSdbMgmt.mutex);
 }
 
-void sdbIncRef(void *handle, void *pObj) {
-  if (pObj == NULL || handle == NULL) return;
+void sdbIncRef(void *tparam, void *pRow) {
+  if (pRow == NULL || tparam == NULL) return;
 
-  SSdbTable *pTable = handle;
-  int32_t *  pRefCount = (int32_t *)(pObj + pTable->refCountPos);
+  SSdbTable *pTable = tparam;
+  int32_t *  pRefCount = (int32_t *)(pRow + pTable->refCountPos);
   int32_t    refCount = atomic_add_fetch_32(pRefCount, 1);
-  sdbTrace("add ref to table:%s record:%p:%s:%d", pTable->tableName, pObj, sdbGetKeyStrFromObj(pTable, pObj), refCount);
+  sdbTrace("vgId:1, sdb:%s, inc ref to row:%p:%s:%d", pTable->name, pRow, sdbGetRowStr(pTable, pRow), refCount);
 }
 
-void sdbDecRef(void *handle, void *pObj) {
-  if (pObj == NULL || handle == NULL) return;
+void sdbDecRef(void *tparam, void *pRow) {
+  if (pRow == NULL || tparam == NULL) return;
 
-  SSdbTable *pTable = handle;
-  int32_t *  pRefCount = (int32_t *)(pObj + pTable->refCountPos);
+  SSdbTable *pTable = tparam;
+  int32_t *  pRefCount = (int32_t *)(pRow + pTable->refCountPos);
   int32_t    refCount = atomic_sub_fetch_32(pRefCount, 1);
-  sdbTrace("def ref of table:%s record:%p:%s:%d", pTable->tableName, pObj, sdbGetKeyStrFromObj(pTable, pObj), refCount);
+  sdbTrace("vgId:1, sdb:%s, dec ref to row:%p:%s:%d", pTable->name, pRow, sdbGetRowStr(pTable, pRow), refCount);
 
-  int32_t *updateEnd = pObj + pTable->refCountPos - 4;
+  int32_t *updateEnd = pRow + pTable->refCountPos - 4;
   if (refCount <= 0 && *updateEnd) {
-    sdbTrace("table:%s, record:%p:%s:%d is destroyed", pTable->tableName, pObj, sdbGetKeyStrFromObj(pTable, pObj), refCount);
-    SSdbOper oper = {.pObj = pObj};
-    (*pTable->destroyFp)(&oper);
+    sdbTrace("vgId:1, sdb:%s, row:%p:%s:%d destroyed", pTable->name, pRow, sdbGetRowStr(pTable, pRow), refCount);
+    SSdbRow row = {.pObj = pRow};
+    (*pTable->fpDestroy)(&row);
   }
 }
 
@@ -487,12 +470,12 @@ static void *sdbGetRowMetaFromObj(SSdbTable *pTable, void *key) {
   return sdbGetRowMeta(pTable, sdbGetObjKey(pTable, key));
 }
 
-void *sdbGetRow(void *handle, void *key) {
-  SSdbTable *pTable = handle;
+void *sdbGetRow(void *tparam, void *key) {
+  SSdbTable *pTable = tparam;
 
   pthread_mutex_lock(&pTable->mutex);
-  void *pRow = sdbGetRowMeta(handle, key);
-  if (pRow) sdbIncRef(handle, pRow);
+  void *pRow = sdbGetRowMeta(pTable, key);
+  if (pRow) sdbIncRef(pTable, pRow);
   pthread_mutex_unlock(&pTable->mutex);
 
   return pRow;
@@ -502,8 +485,8 @@ static void *sdbGetRowFromObj(SSdbTable *pTable, void *key) {
   return sdbGetRow(pTable, sdbGetObjKey(pTable, key));
 }
 
-static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) {
-  void *  key = sdbGetObjKey(pTable, pOper->pObj);
+static int32_t sdbInsertHash(SSdbTable *pTable, SSdbRow *pRow) {
+  void *  key = sdbGetObjKey(pTable, pRow->pObj);
   int32_t keySize = sizeof(int32_t);
 
   if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) {
@@ -511,43 +494,43 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) {
   }
 
   pthread_mutex_lock(&pTable->mutex);
-  taosHashPut(pTable->iHandle, key, keySize, &pOper->pObj, sizeof(int64_t));
+  taosHashPut(pTable->iHandle, key, keySize, &pRow->pObj, sizeof(int64_t));
   pthread_mutex_unlock(&pTable->mutex);
 
-  sdbIncRef(pTable, pOper->pObj);
+  sdbIncRef(pTable, pRow->pObj);
   atomic_add_fetch_32(&pTable->numOfRows, 1);
 
   if (pTable->keyType == SDB_KEY_AUTO) {
-    pTable->autoIndex = MAX(pTable->autoIndex, *((uint32_t *)pOper->pObj));
+    pTable->autoIndex = MAX(pTable->autoIndex, *((uint32_t *)pRow->pObj));
   } else {
     atomic_add_fetch_32(&pTable->autoIndex, 1);
   }
 
-  sdbDebug("table:%s, insert record:%s to hash, rowSize:%d numOfRows:%" PRId64 ", msg:%p", pTable->tableName,
-           sdbGetKeyStrFromObj(pTable, pOper->pObj), pOper->rowSize, pTable->numOfRows, pOper->pMsg);
+  sdbDebug("vgId:1, sdb:%s, insert key:%s to hash, rowSize:%d rows:%" PRId64 ", msg:%p", pTable->name,
+           sdbGetRowStr(pTable, pRow->pObj), pRow->rowSize, pTable->numOfRows, pRow->pMsg);
 
-  int32_t code = (*pTable->insertFp)(pOper);
+  int32_t code = (*pTable->fpInsert)(pRow);
   if (code != TSDB_CODE_SUCCESS) {
-    sdbError("table:%s, failed to insert record:%s to hash, remove it", pTable->tableName,
-             sdbGetKeyStrFromObj(pTable, pOper->pObj));
-    sdbDeleteHash(pTable, pOper);
+    sdbError("vgId:1, sdb:%s, failed to insert key:%s to hash, remove it", pTable->name,
+             sdbGetRowStr(pTable, pRow->pObj));
+    sdbDeleteHash(pTable, pRow);
   }
 
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) {
-  int32_t *updateEnd = pOper->pObj + pTable->refCountPos - 4;
+static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow) {
+  int32_t *updateEnd = pRow->pObj + pTable->refCountPos - 4;
   bool set = atomic_val_compare_exchange_32(updateEnd, 0, 1) == 0;
   if (!set) {
-    sdbError("table:%s, failed to delete record:%s from hash, for it already removed", pTable->tableName,
-             sdbGetKeyStrFromObj(pTable, pOper->pObj));
+    sdbError("vgId:1, sdb:%s, failed to delete key:%s from hash, for it already removed", pTable->name,
+             sdbGetRowStr(pTable, pRow->pObj));
     return TSDB_CODE_MND_SDB_OBJ_NOT_THERE;
   }
 
-  (*pTable->deleteFp)(pOper);
+  (*pTable->fpDelete)(pRow);
   
-  void *  key = sdbGetObjKey(pTable, pOper->pObj);
+  void *  key = sdbGetObjKey(pTable, pRow->pObj);
   int32_t keySize = sizeof(int32_t);
   if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) {
     keySize = strlen((char *)key);
@@ -559,338 +542,263 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) {
 
   atomic_sub_fetch_32(&pTable->numOfRows, 1);
 
-  sdbDebug("table:%s, delete record:%s from hash, numOfRows:%" PRId64 ", msg:%p", pTable->tableName,
-           sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, pOper->pMsg);
+  sdbDebug("vgId:1, sdb:%s, delete key:%s from hash, numOfRows:%" PRId64 ", msg:%p", pTable->name,
+           sdbGetRowStr(pTable, pRow->pObj), pTable->numOfRows, pRow->pMsg);
+
+  sdbDecRef(pTable, pRow->pObj);
+
+  return TSDB_CODE_SUCCESS;
+}
 
-  sdbDecRef(pTable, pOper->pObj);
+static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbRow *pRow) {
+  sdbDebug("vgId:1, sdb:%s, update key:%s in hash, numOfRows:%" PRId64 ", msg:%p", pTable->name,
+           sdbGetRowStr(pTable, pRow->pObj), pTable->numOfRows, pRow->pMsg);
 
+  (*pTable->fpUpdate)(pRow);
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbOper *pOper) {
-  sdbDebug("table:%s, update record:%s in hash, numOfRows:%" PRId64 ", msg:%p", pTable->tableName,
-           sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, pOper->pMsg);
+static int32_t sdbPerformInsertAction(SWalHead *pHead, SSdbTable *pTable) {
+  SSdbRow row = {.rowSize = pHead->len, .rowData = pHead->cont, .pTable = pTable};
+  (*pTable->fpDecode)(&row);
+  return sdbInsertHash(pTable, &row);
+}
 
-  (*pTable->updateFp)(pOper);
+static int32_t sdbPerformDeleteAction(SWalHead *pHead, SSdbTable *pTable) {
+  void *pObj = sdbGetRowMeta(pTable, pHead->cont);
+  if (pObj == NULL) {
+    sdbDebug("vgId:1, sdb:%s, object:%s not exist in hash, ignore delete action", pTable->name,
+             sdbGetKeyStr(pTable, pHead->cont));
     return TSDB_CODE_SUCCESS;
+  }
+  SSdbRow row = {.pTable = pTable, .pObj = pObj};
+  return sdbDeleteHash(pTable, &row);
 }
 
-static int sdbWrite(void *param, void *data, int32_t type, void *pMsg) {
-  SSdbOper *pOper = param;
-  SWalHead *pHead = data;
+static int32_t sdbPerformUpdateAction(SWalHead *pHead, SSdbTable *pTable) {
+  void *pObj = sdbGetRowMeta(pTable, pHead->cont);
+  if (pObj == NULL) {
+    sdbDebug("vgId:1, sdb:%s, object:%s not exist in hash, ignore update action", pTable->name,
+             sdbGetKeyStr(pTable, pHead->cont));
+    return TSDB_CODE_SUCCESS;
+  }
+  SSdbRow row = {.rowSize = pHead->len, .rowData = pHead->cont, .pTable = pTable};
+  (*pTable->fpDecode)(&row);
+  return sdbUpdateHash(pTable, &row);
+}
+
+static int32_t sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void *unused) {
+  SSdbRow *pRow = wparam;
+  SWalHead *pHead = hparam;
   int32_t tableId = pHead->msgType / 10;
   int32_t action = pHead->msgType % 10;
 
   SSdbTable *pTable = sdbGetTableFromId(tableId);
   assert(pTable != NULL);
 
-  pthread_mutex_lock(&tsSdbObj.mutex);
+  if (qtype == TAOS_QTYPE_QUERY) return sdbPerformDeleteAction(pHead, pTable);
+
+  pthread_mutex_lock(&tsSdbMgmt.mutex);
   
   if (pHead->version == 0) {
     // assign version
-    tsSdbObj.version++;
-    pHead->version = tsSdbObj.version;
+    tsSdbMgmt.version++;
+    pHead->version = tsSdbMgmt.version;
   } else {
     // for data from WAL or forward, version may be smaller
-    if (pHead->version <= tsSdbObj.version) {
-      pthread_mutex_unlock(&tsSdbObj.mutex);
-      sdbDebug("table:%s, failed to restore %s record:%s from source(%d), ver:%" PRId64 " too large, sdb ver:%" PRId64,
-               pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), type, pHead->version, tsSdbObj.version);
+    if (pHead->version <= tsSdbMgmt.version) {
+      pthread_mutex_unlock(&tsSdbMgmt.mutex);
+      sdbDebug("vgId:1, sdb:%s, failed to restore %s key:%s from source(%d), hver:%" PRIu64 " too large, mver:%" PRIu64,
+               pTable->name, actStr[action], sdbGetKeyStr(pTable, pHead->cont), qtype, pHead->version, tsSdbMgmt.version);
       return TSDB_CODE_SUCCESS;
-    } else if (pHead->version != tsSdbObj.version + 1) {
-      pthread_mutex_unlock(&tsSdbObj.mutex);
-      sdbError("table:%s, failed to restore %s record:%s from source(%d), ver:%" PRId64 " too large, sdb ver:%" PRId64,
-               pTable->tableName, sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), type, pHead->version, tsSdbObj.version);
+    } else if (pHead->version != tsSdbMgmt.version + 1) {
+      pthread_mutex_unlock(&tsSdbMgmt.mutex);
+      sdbError("vgId:1, sdb:%s, failed to restore %s key:%s from source(%d), hver:%" PRIu64 " too large, mver:%" PRIu64,
+               pTable->name, actStr[action], sdbGetKeyStr(pTable, pHead->cont), qtype, pHead->version, tsSdbMgmt.version);
       return TSDB_CODE_SYN_INVALID_VERSION;
     } else {
-      tsSdbObj.version = pHead->version;
+      tsSdbMgmt.version = pHead->version;
     }
   }
 
-  int32_t code = walWrite(tsSdbObj.wal, pHead);
+  int32_t code = walWrite(tsSdbMgmt.wal, pHead);
   if (code < 0) {
-    pthread_mutex_unlock(&tsSdbObj.mutex);
+    pthread_mutex_unlock(&tsSdbMgmt.mutex);
     return code;
   }
 
-  pthread_mutex_unlock(&tsSdbObj.mutex);
+  pthread_mutex_unlock(&tsSdbMgmt.mutex);
 
-  // from app, oper is created
-  if (pOper != NULL) {
+  // from app, row is created
+  if (pRow != NULL) {
     // forward to peers
-    pOper->processedCount = 0;
-    int32_t syncCode = syncForwardToPeer(tsSdbObj.sync, pHead, pOper, TAOS_QTYPE_RPC);
-    if (syncCode <= 0) pOper->processedCount = 1;
+    pRow->processedCount = 0;
+    int32_t syncCode = syncForwardToPeer(tsSdbMgmt.sync, pHead, pRow, TAOS_QTYPE_RPC);
+    if (syncCode <= 0) pRow->processedCount = 1;
 
     if (syncCode < 0) {
-      sdbError("table:%s, failed to forward request, result:%s action:%s record:%s ver:%" PRId64 ", msg:%p", pTable->tableName,
-               tstrerror(syncCode), sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version, pOper->pMsg);
+      sdbError("vgId:1, sdb:%s, failed to forward req since %s action:%s key:%s hver:%" PRIu64 ", msg:%p", pTable->name,
+               tstrerror(syncCode), actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version, pRow->pMsg);
     } else if (syncCode > 0) {
-      sdbDebug("table:%s, forward request is sent, action:%s record:%s ver:%" PRId64 ", msg:%p", pTable->tableName,
-               sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version, pOper->pMsg);
+      sdbDebug("vgId:1, sdb:%s, forward req is sent, action:%s key:%s hver:%" PRIu64 ", msg:%p", pTable->name,
+               actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version, pRow->pMsg);
     } else {
-      sdbTrace("table:%s, no need to send fwd request, action:%s record:%s ver:%" PRId64 ", msg:%p", pTable->tableName,
-               sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version, pOper->pMsg);
+      sdbTrace("vgId:1, sdb:%s, no need to send fwd req, action:%s key:%s hver:%" PRIu64 ", msg:%p", pTable->name,
+               actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version, pRow->pMsg);
     }
     return syncCode;
   }
 
-  sdbDebug("table:%s, record from wal/fwd is disposed, action:%s record:%s ver:%" PRId64, pTable->tableName,
-           sdbGetActionStr(action), sdbGetKeyStr(pTable, pHead->cont), pHead->version);
+  sdbDebug("vgId:1, sdb:%s, record from wal/fwd is disposed, action:%s key:%s hver:%" PRIu64, pTable->name,
+           actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version);
 
   // even it is WAL/FWD, it shall be called to update version in sync
-  syncForwardToPeer(tsSdbObj.sync, pHead, pOper, TAOS_QTYPE_RPC);
+  syncForwardToPeer(tsSdbMgmt.sync, pHead, pRow, TAOS_QTYPE_RPC);
 
-  // from wal or forward msg, oper not created, should add into hash
+  // from wal or forward msg, row not created, should add into hash
   if (action == SDB_ACTION_INSERT) {
-    SSdbOper oper = {.rowSize = pHead->len, .rowData = pHead->cont, .table = pTable};
-    code = (*pTable->decodeFp)(&oper);
-    return sdbInsertHash(pTable, &oper);
+    return sdbPerformInsertAction(pHead, pTable);
   } else if (action == SDB_ACTION_DELETE) {
-    void *pRow = sdbGetRowMeta(pTable, pHead->cont);
-    if (pRow == NULL) {
-      sdbDebug("table:%s, object:%s not exist in hash, ignore delete action", pTable->tableName,
-               sdbGetKeyStr(pTable, pHead->cont));
+    if (qtype == TAOS_QTYPE_FWD) {
+      // Drop database/stable may take a long time and cause a timeout, so we confirm first then reput it into queue
+      sdbWriteFwdToQueue(1, hparam, TAOS_QTYPE_QUERY, unused);
       return TSDB_CODE_SUCCESS;
+    } else {
+      return sdbPerformDeleteAction(pHead, pTable);
     }
-    SSdbOper oper = {.table = pTable, .pObj = pRow};
-    return sdbDeleteHash(pTable, &oper);
   } else if (action == SDB_ACTION_UPDATE) {
-    void *pRow = sdbGetRowMeta(pTable, pHead->cont);
-    if (pRow == NULL) {
-      sdbDebug("table:%s, object:%s not exist in hash, ignore update action", pTable->tableName,
-               sdbGetKeyStr(pTable, pHead->cont));
-      return TSDB_CODE_SUCCESS;
-    }
-    SSdbOper oper = {.rowSize = pHead->len, .rowData = pHead->cont, .table = pTable};
-    code = (*pTable->decodeFp)(&oper);
-    return sdbUpdateHash(pTable, &oper);
+    return sdbPerformUpdateAction(pHead, pTable);
   } else {
     return TSDB_CODE_MND_INVALID_MSG_TYPE;
   }
 }
 
-int32_t sdbInsertRow(SSdbOper *pOper) {
-  SSdbTable *pTable = (SSdbTable *)pOper->table;
+int32_t sdbInsertRow(SSdbRow *pRow) {
+  SSdbTable *pTable = pRow->pTable;
   if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE;
 
-  if (sdbGetRowFromObj(pTable, pOper->pObj)) {
-    sdbError("table:%s, failed to insert record:%s, already exist", pTable->tableName,
-             sdbGetKeyStrFromObj(pTable, pOper->pObj));
-    sdbDecRef(pTable, pOper->pObj);
+  if (sdbGetRowFromObj(pTable, pRow->pObj)) {
+    sdbError("vgId:1, sdb:%s, failed to insert:%s since it exist", pTable->name, sdbGetRowStr(pTable, pRow->pObj));
+    sdbDecRef(pTable, pRow->pObj);
     return TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE;
   }
 
   if (pTable->keyType == SDB_KEY_AUTO) {
-    *((uint32_t *)pOper->pObj) = atomic_add_fetch_32(&pTable->autoIndex, 1);
+    *((uint32_t *)pRow->pObj) = atomic_add_fetch_32(&pTable->autoIndex, 1);
 
     // let vgId increase from 2
-    if (pTable->autoIndex == 1 && strcmp(pTable->tableName, "vgroups") == 0) {
-      *((uint32_t *)pOper->pObj) = atomic_add_fetch_32(&pTable->autoIndex, 1);
+    if (pTable->autoIndex == 1 && pTable->id == SDB_TABLE_VGROUP) {
+      *((uint32_t *)pRow->pObj) = atomic_add_fetch_32(&pTable->autoIndex, 1);
     }
   }
 
-  int32_t code = sdbInsertHash(pTable, pOper);
+  int32_t code = sdbInsertHash(pTable, pRow);
   if (code != TSDB_CODE_SUCCESS) {
-    sdbError("table:%s, failed to insert into hash", pTable->tableName);
+    sdbError("vgId:1, sdb:%s, failed to insert:%s into hash", pTable->name, sdbGetRowStr(pTable, pRow->pObj));
     return code;
   }
 
   // just insert data into memory
-  if (pOper->type != SDB_OPER_GLOBAL) {
+  if (pRow->type != SDB_OPER_GLOBAL) {
     return TSDB_CODE_SUCCESS;
   }
 
-  if (pOper->reqFp) {
-    return (*pOper->reqFp)(pOper->pMsg);
+  if (pRow->fpReq) {
+    return (*pRow->fpReq)(pRow->pMsg);
   } else {
-    return sdbInsertRowImp(pOper);
-  }
-}
-
-int32_t sdbInsertRowImp(SSdbOper *pOper) {
-  SSdbTable *pTable = (SSdbTable *)pOper->table;
-  if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE;
-
-  int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK;
-  SSdbOper *pNewOper = taosAllocateQitem(size);
-
-  SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper) + SDB_SYNC_HACK;
-  pHead->version = 0;
-  pHead->len = pOper->rowSize;
-  pHead->msgType = pTable->tableId * 10 + SDB_ACTION_INSERT;
-
-  pOper->rowData = pHead->cont;
-  (*pTable->encodeFp)(pOper);
-  pHead->len = pOper->rowSize;
-
-  memcpy(pNewOper, pOper, sizeof(SSdbOper));
-
-  if (pNewOper->pMsg != NULL) {
-    sdbDebug("app:%p:%p, table:%s record:%p:%s, insert action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle,
-             pNewOper->pMsg, pTable->tableName, pOper->pObj, sdbGetKeyStrFromObj(pTable, pOper->pObj));
+    return sdbWriteRowToQueue(pRow, SDB_ACTION_INSERT);
   }
-
-  sdbIncRef(pNewOper->table, pNewOper->pObj);
-  taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper);
-
-  return TSDB_CODE_MND_ACTION_IN_PROGRESS;
 }
 
-bool sdbCheckRowDeleted(void *pTableInput, void *pRow) {
-  SSdbTable *pTable = pTableInput;
+bool sdbCheckRowDeleted(void *tparam, void *pRow) {
+  SSdbTable *pTable = tparam;
   if (pTable == NULL) return false;
 
   int32_t *updateEnd = pRow + pTable->refCountPos - 4;
   return atomic_val_compare_exchange_32(updateEnd, 1, 1) == 1;
 }
 
-int32_t sdbDeleteRow(SSdbOper *pOper) {
-  SSdbTable *pTable = (SSdbTable *)pOper->table;
+int32_t sdbDeleteRow(SSdbRow *pRow) {
+  SSdbTable *pTable = pRow->pTable;
   if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE;
 
-  void *pRow = sdbGetRowMetaFromObj(pTable, pOper->pObj);
-  if (pRow == NULL) {
-    sdbDebug("table:%s, record is not there, delete failed", pTable->tableName);
+  void *pObj = sdbGetRowMetaFromObj(pTable, pRow->pObj);
+  if (pObj == NULL) {
+    sdbDebug("vgId:1, sdb:%s, record is not there, delete failed", pTable->name);
     return TSDB_CODE_MND_SDB_OBJ_NOT_THERE;
   }
 
-  sdbIncRef(pTable, pOper->pObj);
-
-  int32_t code = sdbDeleteHash(pTable, pOper);
+  int32_t code = sdbDeleteHash(pTable, pRow);
   if (code != TSDB_CODE_SUCCESS) {
-    sdbError("table:%s, failed to delete from hash", pTable->tableName);
-    sdbDecRef(pTable, pOper->pObj);
+    sdbError("vgId:1, sdb:%s, failed to delete from hash", pTable->name);
     return code;
   }
 
   // just delete data from memory
-  if (pOper->type != SDB_OPER_GLOBAL) {
-    sdbDecRef(pTable, pOper->pObj);
+  if (pRow->type != SDB_OPER_GLOBAL) {
     return TSDB_CODE_SUCCESS;
   }
 
-  if (pOper->reqFp) {
-    return (*pOper->reqFp)(pOper->pMsg);
+  if (pRow->fpReq) {
+    return (*pRow->fpReq)(pRow->pMsg);
   } else {
-    return sdbDeleteRowImp(pOper);
-  }
-}
-
-int32_t sdbDeleteRowImp(SSdbOper *pOper) {
-  SSdbTable *pTable = (SSdbTable *)pOper->table;
-  if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE;
-
-  int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK;
-  SSdbOper *pNewOper = taosAllocateQitem(size);
-
-  SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper) + SDB_SYNC_HACK;
-  pHead->version = 0;
-  pHead->msgType = pTable->tableId * 10 + SDB_ACTION_DELETE;
-
-  pOper->rowData = pHead->cont;
-  (*pTable->encodeFp)(pOper);
-  pHead->len = pOper->rowSize;
-
-  memcpy(pNewOper, pOper, sizeof(SSdbOper));
-
-  if (pNewOper->pMsg != NULL) {
-    sdbDebug("app:%p:%p, table:%s record:%p:%s, delete action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle,
-             pNewOper->pMsg, pTable->tableName, pOper->pObj, sdbGetKeyStrFromObj(pTable, pOper->pObj));
+    return sdbWriteRowToQueue(pRow, SDB_ACTION_DELETE);
   }
-
-  taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper);
-
-  return TSDB_CODE_MND_ACTION_IN_PROGRESS;
 }
 
-int32_t sdbUpdateRow(SSdbOper *pOper) {
-  SSdbTable *pTable = (SSdbTable *)pOper->table;
+int32_t sdbUpdateRow(SSdbRow *pRow) {
+  SSdbTable *pTable = pRow->pTable;
   if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE;
 
-  void *pRow = sdbGetRowMetaFromObj(pTable, pOper->pObj);
-  if (pRow == NULL) {
-    sdbDebug("table:%s, record is not there, update failed", pTable->tableName);
+  void *pObj = sdbGetRowMetaFromObj(pTable, pRow->pObj);
+  if (pObj == NULL) {
+    sdbDebug("vgId:1, sdb:%s, record is not there, update failed", pTable->name);
     return TSDB_CODE_MND_SDB_OBJ_NOT_THERE;
   }
 
-  int32_t code = sdbUpdateHash(pTable, pOper);
+  int32_t code = sdbUpdateHash(pTable, pRow);
   if (code != TSDB_CODE_SUCCESS) {
-    sdbError("table:%s, failed to update hash", pTable->tableName);
+    sdbError("vgId:1, sdb:%s, failed to update hash", pTable->name);
     return code;
   }
 
   // just update data in memory
-  if (pOper->type != SDB_OPER_GLOBAL) {
+  if (pRow->type != SDB_OPER_GLOBAL) {
     return TSDB_CODE_SUCCESS;
   }
 
-  if (pOper->reqFp) {
-    return (*pOper->reqFp)(pOper->pMsg);
+  if (pRow->fpReq) {
+    return (*pRow->fpReq)(pRow->pMsg);
   } else {
-    return sdbUpdateRowImp(pOper);
+    return sdbWriteRowToQueue(pRow, SDB_ACTION_UPDATE);
   }
 }
 
-int32_t sdbUpdateRowImp(SSdbOper *pOper) {
-  SSdbTable *pTable = (SSdbTable *)pOper->table;
-  if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE;
-
-  int32_t size = sizeof(SSdbOper) + sizeof(SWalHead) + pTable->maxRowSize + SDB_SYNC_HACK;
-  SSdbOper *pNewOper = taosAllocateQitem(size);
-
-  SWalHead *pHead = (void *)pNewOper + sizeof(SSdbOper) + SDB_SYNC_HACK;
-  pHead->version = 0;
-  pHead->msgType = pTable->tableId * 10 + SDB_ACTION_UPDATE;
-
-  pOper->rowData = pHead->cont;
-  (*pTable->encodeFp)(pOper);
-  pHead->len = pOper->rowSize;
-
-  memcpy(pNewOper, pOper, sizeof(SSdbOper));
-
-  if (pNewOper->pMsg != NULL) {
-    sdbDebug("app:%p:%p, table:%s record:%p:%s, update action is add to sdb queue", pNewOper->pMsg->rpcMsg.ahandle,
-             pNewOper->pMsg, pTable->tableName, pOper->pObj, sdbGetKeyStrFromObj(pTable, pOper->pObj));
-  }
-
-  sdbIncRef(pNewOper->table, pNewOper->pObj);
-  taosWriteQitem(tsSdbWriteQueue, TAOS_QTYPE_RPC, pNewOper);
-
-  return TSDB_CODE_MND_ACTION_IN_PROGRESS;
-}
-
-void *sdbFetchRow(void *handle, void *pNode, void **ppRow) {
-  SSdbTable *pTable = (SSdbTable *)handle;
+void *sdbFetchRow(void *tparam, void *pIter, void **ppRow) {
+  SSdbTable *pTable = tparam;
   *ppRow = NULL;
   if (pTable == NULL) return NULL;
 
-  SHashMutableIterator *pIter = pNode;
-  if (pIter == NULL) {
-    pIter = taosHashCreateIter(pTable->iHandle);
-  }
-
-  if (!taosHashIterNext(pIter)) {
-    taosHashDestroyIter(pIter);
-    return NULL;
-  }
+  pIter = taosHashIterate(pTable->iHandle, pIter);
+  if (pIter == NULL) return NULL;
 
-  void **ppMetaRow = taosHashIterGet(pIter);
+  void **ppMetaRow = pIter;
   if (ppMetaRow == NULL) {
-    taosHashDestroyIter(pIter);
+    taosHashCancelIterate(pTable->iHandle, pIter);
     return NULL;
   }
 
   *ppRow = *ppMetaRow;
-  sdbIncRef(handle, *ppMetaRow);
+  sdbIncRef(pTable, *ppMetaRow);
 
   return pIter;
 }
 
-void sdbFreeIter(void *pIter) {
-  if (pIter != NULL) {
-    taosHashDestroyIter(pIter);
-  }
+void sdbFreeIter(void *tparam, void *pIter) {
+  SSdbTable *pTable = tparam;
+  if (pTable == NULL || pIter == NULL) return;
+
+  taosHashCancelIterate(pTable->iHandle, pIter);
 }
 
 void *sdbOpenTable(SSdbTableDesc *pDesc) {
@@ -899,28 +807,28 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) {
   if (pTable == NULL) return NULL;
 
   pthread_mutex_init(&pTable->mutex, NULL);
-  tstrncpy(pTable->tableName, pDesc->tableName, SDB_TABLE_LEN);
+  tstrncpy(pTable->name, pDesc->name, SDB_TABLE_LEN);
   pTable->keyType      = pDesc->keyType;
-  pTable->tableId      = pDesc->tableId;
+  pTable->id           = pDesc->id;
   pTable->hashSessions = pDesc->hashSessions;
   pTable->maxRowSize   = pDesc->maxRowSize;
   pTable->refCountPos  = pDesc->refCountPos;
-  pTable->insertFp     = pDesc->insertFp;
-  pTable->deleteFp     = pDesc->deleteFp;
-  pTable->updateFp     = pDesc->updateFp;
-  pTable->encodeFp     = pDesc->encodeFp;
-  pTable->decodeFp     = pDesc->decodeFp;
-  pTable->destroyFp    = pDesc->destroyFp;
-  pTable->restoredFp   = pDesc->restoredFp;
+  pTable->fpInsert     = pDesc->fpInsert;
+  pTable->fpDelete     = pDesc->fpDelete;
+  pTable->fpUpdate     = pDesc->fpUpdate;
+  pTable->fpEncode     = pDesc->fpEncode;
+  pTable->fpDecode     = pDesc->fpDecode;
+  pTable->fpDestroy    = pDesc->fpDestroy;
+  pTable->fpRestored   = pDesc->fpRestored;
 
   _hash_fn_t hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT);
   if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) {
     hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
   }
-  pTable->iHandle = taosHashInit(pTable->hashSessions, hashFp, true, true);
+  pTable->iHandle = taosHashInit(pTable->hashSessions, hashFp, true, HASH_ENTRY_LOCK);
 
-  tsSdbObj.numOfTables++;
-  tsSdbObj.tableList[pTable->tableId] = pTable;
+  tsSdbMgmt.numOfTables++;
+  tsSdbMgmt.tableList[pTable->id] = pTable;
   return pTable;
 }
 
@@ -928,87 +836,88 @@ void sdbCloseTable(void *handle) {
   SSdbTable *pTable = (SSdbTable *)handle;
   if (pTable == NULL) return;
   
-  tsSdbObj.numOfTables--;
-  tsSdbObj.tableList[pTable->tableId] = NULL;
+  tsSdbMgmt.numOfTables--;
+  tsSdbMgmt.tableList[pTable->id] = NULL;
 
-  SHashMutableIterator *pIter = taosHashCreateIter(pTable->iHandle);
-  while (taosHashIterNext(pIter)) {
-    void **ppRow = taosHashIterGet(pIter);
+  void *pIter = taosHashIterate(pTable->iHandle, NULL);
+  while (pIter) {
+    void **ppRow = pIter;
+    pIter = taosHashIterate(pTable->iHandle, pIter);
     if (ppRow == NULL) continue;
 
-    SSdbOper oper = {
+    SSdbRow row = {
       .pObj = *ppRow,
-      .table = pTable,
+      .pTable = pTable,
     };
 
-    (*pTable->destroyFp)(&oper);
+    (*pTable->fpDestroy)(&row);
   }
 
-  taosHashDestroyIter(pIter);
+  taosHashCancelIterate(pTable->iHandle, pIter);
   taosHashCleanup(pTable->iHandle);
   pthread_mutex_destroy(&pTable->mutex);
 
-  sdbDebug("table:%s, is closed, numOfTables:%d", pTable->tableName, tsSdbObj.numOfTables);
+  sdbDebug("vgId:1, sdb:%s, is closed, numOfTables:%d", pTable->name, tsSdbMgmt.numOfTables);
   free(pTable);
 }
 
-int32_t sdbInitWriteWorker() {
+static int32_t sdbInitWorker() {
   tsSdbPool.num = 1;
-  tsSdbPool.writeWorker = (SSdbWriteWorker *)calloc(sizeof(SSdbWriteWorker), tsSdbPool.num);
+  tsSdbPool.worker = calloc(sizeof(SSdbWorker), tsSdbPool.num);
 
-  if (tsSdbPool.writeWorker == NULL) return -1;
+  if (tsSdbPool.worker == NULL) return -1;
   for (int32_t i = 0; i < tsSdbPool.num; ++i) {
-    SSdbWriteWorker *pWorker = tsSdbPool.writeWorker + i;
+    SSdbWorker *pWorker = tsSdbPool.worker + i;
     pWorker->workerId = i;
   }
 
-  sdbAllocWriteQueue();
+  sdbAllocQueue();
   
-  mInfo("sdb write is opened");
+  mInfo("vgId:1, sdb write is opened");
   return 0;
 }
 
-void sdbCleanupWriteWorker() {
+static void sdbCleanupWorker() {
   for (int32_t i = 0; i < tsSdbPool.num; ++i) {
-    SSdbWriteWorker *pWorker = tsSdbPool.writeWorker + i;
+    SSdbWorker *pWorker = tsSdbPool.worker + i;
     if (pWorker->thread) {
-      taosQsetThreadResume(tsSdbWriteQset);
+      taosQsetThreadResume(tsSdbWQset);
     }
   }
 
   for (int32_t i = 0; i < tsSdbPool.num; ++i) {
-    SSdbWriteWorker *pWorker = tsSdbPool.writeWorker + i;
+    SSdbWorker *pWorker = tsSdbPool.worker + i;
     if (pWorker->thread) {
       pthread_join(pWorker->thread, NULL);
     }
   }
 
-  sdbFreeWritequeue();
-  tfree(tsSdbPool.writeWorker);
+  sdbFreeQueue();
+  tfree(tsSdbPool.worker);
 
-  mInfo("sdb write is closed");
+  mInfo("vgId:1, sdb write is closed");
 }
 
-int32_t sdbAllocWriteQueue() {
-  tsSdbWriteQueue = taosOpenQueue();
-  if (tsSdbWriteQueue == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
+static int32_t sdbAllocQueue() {
+  tsSdbWQueue = taosOpenQueue();
+  if (tsSdbWQueue == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
 
-  tsSdbWriteQset = taosOpenQset();
-  if (tsSdbWriteQset == NULL) {
-    taosCloseQueue(tsSdbWriteQueue);
+  tsSdbWQset = taosOpenQset();
+  if (tsSdbWQset == NULL) {
+    taosCloseQueue(tsSdbWQueue);
     return TSDB_CODE_MND_OUT_OF_MEMORY;
   }
-  taosAddIntoQset(tsSdbWriteQset, tsSdbWriteQueue, NULL);
+  taosAddIntoQset(tsSdbWQset, tsSdbWQueue, NULL);
 
-  tsSdbWriteQall = taosAllocateQall();
-  if (tsSdbWriteQall == NULL) {
-    taosCloseQset(tsSdbWriteQset);
-    taosCloseQueue(tsSdbWriteQueue);
+  tsSdbWQall = taosAllocateQall();
+  if (tsSdbWQall == NULL) {
+    taosCloseQset(tsSdbWQset);
+    taosCloseQueue(tsSdbWQueue);
     return TSDB_CODE_MND_OUT_OF_MEMORY;
   }
   
   for (int32_t i = 0; i < tsSdbPool.num; ++i) {
-    SSdbWriteWorker *pWorker = tsSdbPool.writeWorker + i;
+    SSdbWorker *pWorker = tsSdbPool.worker + i;
     pWorker->workerId = i;
 
     pthread_attr_t thAttr;
@@ -1017,9 +926,9 @@ int32_t sdbAllocWriteQueue() {
 
     if (pthread_create(&pWorker->thread, &thAttr, sdbWorkerFp, pWorker) != 0) {
       mError("failed to create thread to process sdb write queue, reason:%s", strerror(errno));
-      taosFreeQall(tsSdbWriteQall);
-      taosCloseQset(tsSdbWriteQset);
-      taosCloseQueue(tsSdbWriteQueue);
+      taosFreeQall(tsSdbWQall);
+      taosCloseQset(tsSdbWQset);
+      taosCloseQueue(tsSdbWQueue);
       return TSDB_CODE_MND_OUT_OF_MEMORY;
     }
 
@@ -1027,85 +936,128 @@ int32_t sdbAllocWriteQueue() {
     mDebug("sdb write worker:%d is launched, total:%d", pWorker->workerId, tsSdbPool.num);
   }
 
-  mDebug("sdb write queue:%p is allocated", tsSdbWriteQueue);
+  mDebug("sdb write queue:%p is allocated", tsSdbWQueue);
   return TSDB_CODE_SUCCESS;
 }
 
-void sdbFreeWritequeue() {
-  taosCloseQueue(tsSdbWriteQueue);
-  taosFreeQall(tsSdbWriteQall);
-  taosCloseQset(tsSdbWriteQset);
-  tsSdbWriteQall = NULL;
-  tsSdbWriteQset = NULL;
-  tsSdbWriteQueue = NULL;
+static void sdbFreeQueue() {
+  taosCloseQueue(tsSdbWQueue);
+  taosFreeQall(tsSdbWQall);
+  taosCloseQset(tsSdbWQset);
+  tsSdbWQall = NULL;
+  tsSdbWQset = NULL;
+  tsSdbWQueue = NULL;
 }
 
-int32_t sdbWriteToQueue(void *param, void *data, int32_t qtype, void *pMsg) {
-  SWalHead *pHead = data;
-  int32_t   size = sizeof(SWalHead) + pHead->len;
-  SWalHead *pWal = taosAllocateQitem(size);
-  memcpy(pWal, pHead, size);
+static int32_t sdbWriteToQueue(SSdbRow *pRow, int32_t qtype) {
+  SWalHead *pHead = pRow->pHead;
 
-  taosWriteQitem(tsSdbWriteQueue, qtype, pWal);
-  return 0;
+  if (pHead->len > TSDB_MAX_WAL_SIZE) {
+    sdbError("vgId:1, wal len:%d exceeds limit, hver:%" PRIu64, pHead->len, pHead->version);
+    taosFreeQitem(pRow);
+    return TSDB_CODE_WAL_SIZE_LIMIT;
+  }
+
+  int32_t queued = atomic_add_fetch_32(&tsSdbMgmt.queuedMsg, 1);
+  if (queued > MAX_QUEUED_MSG_NUM) {
+    sdbDebug("vgId:1, too many msg:%d in sdb queue, flow control", queued);
+    taosMsleep(1);
+  }
+
+  sdbIncRef(pRow->pTable, pRow->pObj);
+
+  sdbTrace("vgId:1, msg:%p qtype:%s write into to sdb queue, queued:%d", pRow->pMsg, qtypeStr[qtype], queued);
+  taosWriteQitem(tsSdbWQueue, qtype, pRow);
+
+  return TSDB_CODE_MND_ACTION_IN_PROGRESS;
+}
+
+static void sdbFreeFromQueue(SSdbRow *pRow) {
+  int32_t queued = atomic_sub_fetch_32(&tsSdbMgmt.queuedMsg, 1);
+  sdbTrace("vgId:1, msg:%p free from sdb queue, queued:%d", pRow->pMsg, queued);
+
+  sdbDecRef(pRow->pTable, pRow->pObj);
+  taosFreeQitem(pRow);
+}
+
+static int32_t sdbWriteFwdToQueue(int32_t vgId, void *wparam, int32_t qtype, void *rparam) {
+  SWalHead *pHead = wparam;
+
+  int32_t  size = sizeof(SSdbRow) + sizeof(SWalHead) + pHead->len;
+  SSdbRow *pRow = taosAllocateQitem(size);
+  if (pRow == NULL) {
+    return TSDB_CODE_VND_OUT_OF_MEMORY;
+  }
+
+  memcpy(pRow->pHead, pHead, sizeof(SWalHead) + pHead->len);
+  pRow->rowData = pRow->pHead->cont;
+
+  return sdbWriteToQueue(pRow, qtype);
+}
+
+static int32_t sdbWriteRowToQueue(SSdbRow *pInputRow, int32_t action) {
+  SSdbTable *pTable = pInputRow->pTable;
+  if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE;
+
+  int32_t  size = sizeof(SSdbRow) + sizeof(SWalHead) + pTable->maxRowSize;
+  SSdbRow *pRow = taosAllocateQitem(size);
+  if (pRow == NULL) {
+    return TSDB_CODE_VND_OUT_OF_MEMORY;
+  }
+
+  memcpy(pRow, pInputRow, sizeof(SSdbRow));
+  pRow->processedCount = 1;
+
+  SWalHead *pHead = pRow->pHead;
+  pRow->rowData = pHead->cont;
+  (*pTable->fpEncode)(pRow);
+
+  pHead->len = pRow->rowSize;
+  pHead->version = 0;
+  pHead->msgType = pTable->id * 10 + action;
+
+  return sdbWriteToQueue(pRow, TAOS_QTYPE_RPC);
 }
 
-static void *sdbWorkerFp(void *param) {
-  SWalHead *pHead;
-  SSdbOper *pOper;
-  int32_t   type;
-  int32_t   numOfMsgs;
-  void *    item;
+int32_t sdbInsertRowToQueue(SSdbRow *pRow) { return sdbWriteRowToQueue(pRow, SDB_ACTION_INSERT); }
+
+static void *sdbWorkerFp(void *pWorker) {
+  SSdbRow *pRow;
+  int32_t  qtype;
   void *   unUsed;
 
   while (1) {
-    numOfMsgs = taosReadAllQitemsFromQset(tsSdbWriteQset, tsSdbWriteQall, &unUsed);
+    int32_t numOfMsgs = taosReadAllQitemsFromQset(tsSdbWQset, tsSdbWQall, &unUsed);
     if (numOfMsgs == 0) {
-      sdbDebug("qset:%p, sdb got no message from qset, exiting", tsSdbWriteQset);
+      sdbDebug("qset:%p, sdb got no message from qset, exiting", tsSdbWQset);
       break;
     }
 
     for (int32_t i = 0; i < numOfMsgs; ++i) {
-      taosGetQitem(tsSdbWriteQall, &type, &item);
-      if (type == TAOS_QTYPE_RPC) {
-        pOper = (SSdbOper *)item;
-        pOper->processedCount = 1;
-        pHead = (void *)pOper + sizeof(SSdbOper) + SDB_SYNC_HACK;
-        if (pOper->pMsg != NULL) {
-          sdbDebug("app:%p:%p, table:%s record:%p:%s ver:%" PRIu64 ", will be processed in sdb queue",
-                   pOper->pMsg->rpcMsg.ahandle, pOper->pMsg, ((SSdbTable *)pOper->table)->tableName, pOper->pObj,
-                   sdbGetKeyStr(pOper->table, pHead->cont), pHead->version);
-        }
-      } else {
-        pHead = (SWalHead *)item;
-        pOper = NULL;
-      }
+      taosGetQitem(tsSdbWQall, &qtype, (void **)&pRow);
+      sdbTrace("vgId:1, msg:%p, row:%p hver:%" PRIu64 ", will be processed in sdb queue", pRow->pMsg, pRow->pObj,
+               pRow->pHead->version);
 
-      int32_t code = sdbWrite(pOper, pHead, type, NULL);
-      if (code > 0) code = 0;
-      if (pOper) {
-        pOper->retCode = code;
-      } else {
-        pHead->len = code;  // hackway
-      }
+      pRow->code = sdbProcessWrite((qtype == TAOS_QTYPE_RPC) ? pRow : NULL, pRow->pHead, qtype, NULL);
+      if (pRow->code > 0) pRow->code = 0;
+
+      sdbTrace("vgId:1, msg:%p is processed in sdb queue, code:%x", pRow->pMsg, pRow->code);
     }
 
-    walFsync(tsSdbObj.wal, true);
+    walFsync(tsSdbMgmt.wal, true);
 
     // browse all items, and process them one by one
-    taosResetQitems(tsSdbWriteQall);
+    taosResetQitems(tsSdbWQall);
     for (int32_t i = 0; i < numOfMsgs; ++i) {
-      taosGetQitem(tsSdbWriteQall, &type, &item);
-
-      if (type == TAOS_QTYPE_RPC) {
-        pOper = (SSdbOper *)item;
-        sdbConfirmForward(NULL, pOper, pOper->retCode);
-      } else if (type == TAOS_QTYPE_FWD) {
-        pHead = (SWalHead *)item;
-        syncConfirmForward(tsSdbObj.sync, pHead->version, pHead->len);
-        taosFreeQitem(item);
+      taosGetQitem(tsSdbWQall, &qtype, (void **)&pRow);
+
+      if (qtype == TAOS_QTYPE_RPC) {
+        sdbConfirmForward(1, pRow, pRow->code);
       } else {
-        taosFreeQitem(item);
+        if (qtype == TAOS_QTYPE_FWD) {
+          syncConfirmForward(tsSdbMgmt.sync, pRow->pHead->version, pRow->code);
+        }
+        sdbFreeFromQueue(pRow);
       }
     }
   }

src/mnode/src/mnodeShow.c

@@ -57,6 +57,7 @@ static void *tsMnodeShowCache = NULL;
 static int32_t tsShowObjIndex = 0;
 static SShowMetaFp     tsMnodeShowMetaFp[TSDB_MGMT_TABLE_MAX]     = {0};
 static SShowRetrieveFp tsMnodeShowRetrieveFp[TSDB_MGMT_TABLE_MAX] = {0};
+static SShowFreeIterFp tsMnodeShowFreeIterFp[TSDB_MGMT_TABLE_MAX] = {0};
 
 int32_t mnodeInitShow() {
   mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_SHOW, mnodeProcessShowMsg);
@@ -85,6 +86,10 @@ void mnodeAddShowRetrieveHandle(uint8_t msgType, SShowRetrieveFp fp) {
   tsMnodeShowRetrieveFp[msgType] = fp;
 }
 
+void mnodeAddShowFreeIterHandle(uint8_t msgType, SShowFreeIterFp fp) {
+  tsMnodeShowFreeIterFp[msgType] = fp;
+}
+
 static char *mnodeGetShowType(int32_t showType) {
   switch (showType) {
     case TSDB_MGMT_TABLE_ACCT:    return "show accounts";
@@ -412,7 +417,9 @@ static void* mnodePutShowObj(SShowObj *pShow) {
 
 static void mnodeFreeShowObj(void *data) {
   SShowObj *pShow = *(SShowObj **)data;
-  sdbFreeIter(pShow->pIter);
+  if (tsMnodeShowFreeIterFp[pShow->type] != NULL && pShow->pIter != NULL) {
+    (*tsMnodeShowFreeIterFp[pShow->type])(pShow->pIter);
+  }
 
   mDebug("%p, show is destroyed, data:%p index:%d", pShow, data, pShow->index);
   tfree(pShow);

src/mnode/src/mnodeTable.c

@@ -99,13 +99,13 @@ static void mnodeDestroyChildTable(SCTableObj *pTable) {
   tfree(pTable);
 }
 
-static int32_t mnodeChildTableActionDestroy(SSdbOper *pOper) {
-  mnodeDestroyChildTable(pOper->pObj);
+static int32_t mnodeChildTableActionDestroy(SSdbRow *pRow) {
+  mnodeDestroyChildTable(pRow->pObj);
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeChildTableActionInsert(SSdbOper *pOper) {
-  SCTableObj *pTable = pOper->pObj;
+static int32_t mnodeChildTableActionInsert(SSdbRow *pRow) {
+  SCTableObj *pTable = pRow->pObj;
 
   SVgObj *pVgroup = mnodeGetVgroup(pTable->vgId);
   if (pVgroup == NULL) {
@@ -153,8 +153,8 @@ static int32_t mnodeChildTableActionInsert(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeChildTableActionDelete(SSdbOper *pOper) {
-  SCTableObj *pTable = pOper->pObj;
+static int32_t mnodeChildTableActionDelete(SSdbRow *pRow) {
+  SCTableObj *pTable = pRow->pObj;
   if (pTable->vgId == 0) {
     return TSDB_CODE_MND_VGROUP_NOT_EXIST;
   }
@@ -189,8 +189,8 @@ static int32_t mnodeChildTableActionDelete(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeChildTableActionUpdate(SSdbOper *pOper) {
-  SCTableObj *pNew = pOper->pObj;
+static int32_t mnodeChildTableActionUpdate(SSdbRow *pRow) {
+  SCTableObj *pNew = pRow->pObj;
   SCTableObj *pTable = mnodeGetChildTable(pNew->info.tableId);
   if (pTable != pNew) {
     void *oldTableId = pTable->info.tableId;    
@@ -216,50 +216,50 @@ static int32_t mnodeChildTableActionUpdate(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeChildTableActionEncode(SSdbOper *pOper) {
-  SCTableObj *pTable = pOper->pObj;
-  assert(pTable != NULL && pOper->rowData != NULL);
+static int32_t mnodeChildTableActionEncode(SSdbRow *pRow) {
+  SCTableObj *pTable = pRow->pObj;
+  assert(pTable != NULL && pRow->rowData != NULL);
 
   int32_t len = strlen(pTable->info.tableId);
   if (len >= TSDB_TABLE_FNAME_LEN) return TSDB_CODE_MND_INVALID_TABLE_ID;
 
-  memcpy(pOper->rowData, pTable->info.tableId, len);
-  memset(pOper->rowData + len, 0, 1);
+  memcpy(pRow->rowData, pTable->info.tableId, len);
+  memset(pRow->rowData + len, 0, 1);
   len++;
 
-  memcpy(pOper->rowData + len, (char*)pTable + sizeof(char *), tsChildTableUpdateSize);
+  memcpy(pRow->rowData + len, (char*)pTable + sizeof(char *), tsChildTableUpdateSize);
   len += tsChildTableUpdateSize;
 
   if (pTable->info.type != TSDB_CHILD_TABLE) {
     int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema);
-    memcpy(pOper->rowData + len, pTable->schema, schemaSize);
+    memcpy(pRow->rowData + len, pTable->schema, schemaSize);
     len += schemaSize;
 
     if (pTable->sqlLen != 0) {
-      memcpy(pOper->rowData + len, pTable->sql, pTable->sqlLen);
+      memcpy(pRow->rowData + len, pTable->sql, pTable->sqlLen);
       len += pTable->sqlLen;
     }
   }
 
-  pOper->rowSize = len;
+  pRow->rowSize = len;
 
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeChildTableActionDecode(SSdbOper *pOper) {
-  assert(pOper->rowData != NULL);
+static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) {
+  assert(pRow->rowData != NULL);
   SCTableObj *pTable = calloc(1, sizeof(SCTableObj));
   if (pTable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
 
-  int32_t len = strlen(pOper->rowData);
+  int32_t len = strlen(pRow->rowData);
   if (len >= TSDB_TABLE_FNAME_LEN) {
     free(pTable);
     return TSDB_CODE_MND_INVALID_TABLE_ID;
   }
-  pTable->info.tableId = strdup(pOper->rowData);
+  pTable->info.tableId = strdup(pRow->rowData);
   len++;
 
-  memcpy((char*)pTable + sizeof(char *), pOper->rowData + len, tsChildTableUpdateSize);
+  memcpy((char*)pTable + sizeof(char *), pRow->rowData + len, tsChildTableUpdateSize);
   len += tsChildTableUpdateSize;
 
   if (pTable->info.type != TSDB_CHILD_TABLE) {
@@ -269,7 +269,7 @@ static int32_t mnodeChildTableActionDecode(SSdbOper *pOper) {
       mnodeDestroyChildTable(pTable);
       return TSDB_CODE_MND_INVALID_TABLE_TYPE;
     }
-    memcpy(pTable->schema, pOper->rowData + len, schemaSize);
+    memcpy(pTable->schema, pRow->rowData + len, schemaSize);
     len += schemaSize;
 
     if (pTable->sqlLen != 0) {
@@ -278,11 +278,11 @@ static int32_t mnodeChildTableActionDecode(SSdbOper *pOper) {
         mnodeDestroyChildTable(pTable);
         return TSDB_CODE_MND_OUT_OF_MEMORY;
       }
-      memcpy(pTable->sql, pOper->rowData + len, pTable->sqlLen);
+      memcpy(pTable->sql, pRow->rowData + len, pTable->sqlLen);
     }
   }
 
-  pOper->pObj = pTable;
+  pRow->pObj = pTable;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -297,7 +297,7 @@ static int32_t mnodeChildTableActionRestored() {
     SDbObj *pDb = mnodeGetDbByTableId(pTable->info.tableId);
     if (pDb == NULL || pDb->status != TSDB_DB_STATUS_READY) {
       mError("ctable:%s, failed to get db or db in dropping, discard it", pTable->info.tableId);
-      SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb};
+      SSdbRow desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .pTable = tsChildTableSdb};
       sdbDeleteRow(&desc);
       mnodeDecTableRef(pTable);
       mnodeDecDbRef(pDb);
@@ -309,7 +309,7 @@ static int32_t mnodeChildTableActionRestored() {
     if (pVgroup == NULL) {
       mError("ctable:%s, failed to get vgId:%d tid:%d, discard it", pTable->info.tableId, pTable->vgId, pTable->tid);
       pTable->vgId = 0;
-      SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb};
+      SSdbRow desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .pTable = tsChildTableSdb};
       sdbDeleteRow(&desc);
       mnodeDecTableRef(pTable);
       continue;
@@ -320,7 +320,7 @@ static int32_t mnodeChildTableActionRestored() {
       mError("ctable:%s, db:%s not match with vgId:%d db:%s sid:%d, discard it",
              pTable->info.tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->tid);
       pTable->vgId = 0;
-      SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb};
+      SSdbRow desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .pTable = tsChildTableSdb};
       sdbDeleteRow(&desc);
       mnodeDecTableRef(pTable);
       continue;
@@ -331,7 +331,7 @@ static int32_t mnodeChildTableActionRestored() {
       if (pSuperTable == NULL) {
         mError("ctable:%s, stable:%" PRIu64 " not exist", pTable->info.tableId, pTable->suid);
         pTable->vgId = 0;
-        SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb};
+        SSdbRow desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .pTable = tsChildTableSdb};
         sdbDeleteRow(&desc);
         mnodeDecTableRef(pTable);
         continue;
@@ -342,8 +342,7 @@ static int32_t mnodeChildTableActionRestored() {
     mnodeDecTableRef(pTable);
   }
 
-  sdbFreeIter(pIter);
-
+  mnodeCancelGetNextChildTable(pIter);
   return 0;
 }
 
@@ -352,19 +351,19 @@ static int32_t mnodeInitChildTables() {
   tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type;
 
   SSdbTableDesc tableDesc = {
-    .tableId      = SDB_TABLE_CTABLE,
-    .tableName    = "ctables",
+    .id           = SDB_TABLE_CTABLE,
+    .name         = "ctables",
     .hashSessions = TSDB_DEFAULT_CTABLES_HASH_SIZE,
     .maxRowSize   = sizeof(SCTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN + TSDB_CQ_SQL_SIZE,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_VAR_STRING,
-    .insertFp     = mnodeChildTableActionInsert,
-    .deleteFp     = mnodeChildTableActionDelete,
-    .updateFp     = mnodeChildTableActionUpdate,
-    .encodeFp     = mnodeChildTableActionEncode,
-    .decodeFp     = mnodeChildTableActionDecode,
-    .destroyFp    = mnodeChildTableActionDestroy,
-    .restoredFp   = mnodeChildTableActionRestored
+    .fpInsert     = mnodeChildTableActionInsert,
+    .fpDelete     = mnodeChildTableActionDelete,
+    .fpUpdate     = mnodeChildTableActionUpdate,
+    .fpEncode     = mnodeChildTableActionEncode,
+    .fpDecode     = mnodeChildTableActionDecode,
+    .fpDestroy    = mnodeChildTableActionDestroy,
+    .fpRestored   = mnodeChildTableActionRestored
   };
 
   tsChildTableSdb = sdbOpenTable(&tableDesc);
@@ -394,7 +393,7 @@ static void mnodeAddTableIntoStable(SSTableObj *pStable, SCTableObj *pCtable) {
   atomic_add_fetch_32(&pStable->numOfTables, 1);
 
   if (pStable->vgHash == NULL) {
-    pStable->vgHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
+    pStable->vgHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
   }
 
   if (pStable->vgHash != NULL) {
@@ -413,7 +412,7 @@ static void mnodeRemoveTableFromStable(SSTableObj *pStable, SCTableObj *pCtable)
 
   SVgObj *pVgroup = mnodeGetVgroup(pCtable->vgId);
   if (pVgroup == NULL) {
-    taosHashRemove(pStable->vgHash, (char *)&pCtable->vgId, sizeof(pCtable->vgId));
+    taosHashRemove(pStable->vgHash, &pCtable->vgId, sizeof(pCtable->vgId));
     mDebug("table:%s, vgId:%d is remove from stable vgList, sizeOfVgList:%d", pStable->info.tableId, pCtable->vgId,
            (int32_t)taosHashGetSize(pStable->vgHash));
   }
@@ -430,13 +429,13 @@ static void mnodeDestroySuperTable(SSTableObj *pStable) {
   tfree(pStable);
 }
 
-static int32_t mnodeSuperTableActionDestroy(SSdbOper *pOper) {
-  mnodeDestroySuperTable(pOper->pObj);
+static int32_t mnodeSuperTableActionDestroy(SSdbRow *pRow) {
+  mnodeDestroySuperTable(pRow->pObj);
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeSuperTableActionInsert(SSdbOper *pOper) {
-  SSTableObj *pStable = pOper->pObj;
+static int32_t mnodeSuperTableActionInsert(SSdbRow *pRow) {
+  SSTableObj *pStable = pRow->pObj;
   SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId);
   if (pDb != NULL && pDb->status == TSDB_DB_STATUS_READY) {
     mnodeAddSuperTableIntoDb(pDb);
@@ -446,8 +445,8 @@ static int32_t mnodeSuperTableActionInsert(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeSuperTableActionDelete(SSdbOper *pOper) {
-  SSTableObj *pStable = pOper->pObj;
+static int32_t mnodeSuperTableActionDelete(SSdbRow *pRow) {
+  SSTableObj *pStable = pRow->pObj;
   SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId);
   if (pDb != NULL) {
     mnodeRemoveSuperTableFromDb(pDb);
@@ -458,8 +457,8 @@ static int32_t mnodeSuperTableActionDelete(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeSuperTableActionUpdate(SSdbOper *pOper) {
-  SSTableObj *pNew = pOper->pObj;
+static int32_t mnodeSuperTableActionUpdate(SSdbRow *pRow) {
+  SSTableObj *pNew = pRow->pObj;
   SSTableObj *pTable = mnodeGetSuperTable(pNew->info.tableId);
   if (pTable != NULL && pTable != pNew) {
     void *oldTableId = pTable->info.tableId;
@@ -483,43 +482,43 @@ static int32_t mnodeSuperTableActionUpdate(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeSuperTableActionEncode(SSdbOper *pOper) {
-  SSTableObj *pStable = pOper->pObj;
-  assert(pOper->pObj != NULL && pOper->rowData != NULL);
+static int32_t mnodeSuperTableActionEncode(SSdbRow *pRow) {
+  SSTableObj *pStable = pRow->pObj;
+  assert(pRow->pObj != NULL && pRow->rowData != NULL);
 
   int32_t len = strlen(pStable->info.tableId);
   if (len >= TSDB_TABLE_FNAME_LEN) len = TSDB_CODE_MND_INVALID_TABLE_ID;
 
-  memcpy(pOper->rowData, pStable->info.tableId, len);
-  memset(pOper->rowData + len, 0, 1);
+  memcpy(pRow->rowData, pStable->info.tableId, len);
+  memset(pRow->rowData + len, 0, 1);
   len++;
 
-  memcpy(pOper->rowData + len, (char*)pStable + sizeof(char *), tsSuperTableUpdateSize);
+  memcpy(pRow->rowData + len, (char*)pStable + sizeof(char *), tsSuperTableUpdateSize);
   len += tsSuperTableUpdateSize;
 
   int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags);
-  memcpy(pOper->rowData + len, pStable->schema, schemaSize);
+  memcpy(pRow->rowData + len, pStable->schema, schemaSize);
   len += schemaSize;
 
-  pOper->rowSize = len;
+  pRow->rowSize = len;
 
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeSuperTableActionDecode(SSdbOper *pOper) {
-  assert(pOper->rowData != NULL);
+static int32_t mnodeSuperTableActionDecode(SSdbRow *pRow) {
+  assert(pRow->rowData != NULL);
   SSTableObj *pStable = (SSTableObj *) calloc(1, sizeof(SSTableObj));
   if (pStable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
 
-  int32_t len = strlen(pOper->rowData);
+  int32_t len = strlen(pRow->rowData);
   if (len >= TSDB_TABLE_FNAME_LEN){
     free(pStable);
     return TSDB_CODE_MND_INVALID_TABLE_ID;
   }
-  pStable->info.tableId = strdup(pOper->rowData);
+  pStable->info.tableId = strdup(pRow->rowData);
   len++;
 
-  memcpy((char*)pStable + sizeof(char *), pOper->rowData + len, tsSuperTableUpdateSize);
+  memcpy((char*)pStable + sizeof(char *), pRow->rowData + len, tsSuperTableUpdateSize);
   len += tsSuperTableUpdateSize;
 
   int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags);
@@ -529,9 +528,9 @@ static int32_t mnodeSuperTableActionDecode(SSdbOper *pOper) {
     return TSDB_CODE_MND_NOT_SUPER_TABLE;
   }
 
-  memcpy(pStable->schema, pOper->rowData + len, schemaSize);
+  memcpy(pStable->schema, pRow->rowData + len, schemaSize);
   
-  pOper->pObj = pStable;
+  pRow->pObj = pStable;
 
   return TSDB_CODE_SUCCESS;
 }
@@ -545,19 +544,19 @@ static int32_t mnodeInitSuperTables() {
   tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type;
 
   SSdbTableDesc tableDesc = {
-    .tableId      = SDB_TABLE_STABLE,
-    .tableName    = "stables",
+    .id           = SDB_TABLE_STABLE,
+    .name         = "stables",
     .hashSessions = TSDB_DEFAULT_STABLES_HASH_SIZE,
     .maxRowSize   = sizeof(SSTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_VAR_STRING,
-    .insertFp     = mnodeSuperTableActionInsert,
-    .deleteFp     = mnodeSuperTableActionDelete,
-    .updateFp     = mnodeSuperTableActionUpdate,
-    .encodeFp     = mnodeSuperTableActionEncode,
-    .decodeFp     = mnodeSuperTableActionDecode,
-    .destroyFp    = mnodeSuperTableActionDestroy,
-    .restoredFp   = mnodeSuperTableActionRestored
+    .fpInsert     = mnodeSuperTableActionInsert,
+    .fpDelete     = mnodeSuperTableActionDelete,
+    .fpUpdate     = mnodeSuperTableActionUpdate,
+    .fpEncode     = mnodeSuperTableActionEncode,
+    .fpDecode     = mnodeSuperTableActionDecode,
+    .fpDestroy    = mnodeSuperTableActionDestroy,
+    .fpRestored   = mnodeSuperTableActionRestored
   };
 
   tsSuperTableSdb = sdbOpenTable(&tableDesc);
@@ -602,10 +601,13 @@ int32_t mnodeInitTables() {
 
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_TABLE, mnodeGetShowTableMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_TABLE, mnodeRetrieveShowTables);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_TABLE, mnodeCancelGetNextChildTable);
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_METRIC, mnodeGetShowSuperTableMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_METRIC, mnodeRetrieveShowSuperTables);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_METRIC, mnodeCancelGetNextSuperTable);
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_STREAMTABLES, mnodeGetStreamTableMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_STREAMTABLES, mnodeRetrieveStreamTables);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_STREAMTABLES, mnodeCancelGetNextChildTable);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -626,14 +628,12 @@ static void *mnodeGetSuperTableByUid(uint64_t uid) {
     pIter = mnodeGetNextSuperTable(pIter, &pStable);
     if (pStable == NULL) break;
     if (pStable->uid == uid) {
-      sdbFreeIter(pIter);
+      mnodeCancelGetNextSuperTable(pIter);
       return pStable;
     }
     mnodeDecTableRef(pStable);
   }
 
-  sdbFreeIter(pIter);
-
   return NULL;
 }
 
@@ -655,10 +655,18 @@ void *mnodeGetNextChildTable(void *pIter, SCTableObj **pTable) {
   return sdbFetchRow(tsChildTableSdb, pIter, (void **)pTable);
 }
 
+void mnodeCancelGetNextChildTable(void *pIter) {
+  sdbFreeIter(tsChildTableSdb, pIter);
+}
+
 void *mnodeGetNextSuperTable(void *pIter, SSTableObj **pTable) {
   return sdbFetchRow(tsSuperTableSdb, pIter, (void **)pTable);
 }
 
+void mnodeCancelGetNextSuperTable(void *pIter) {
+  sdbFreeIter(tsSuperTableSdb, pIter);
+}
+
 void mnodeIncTableRef(void *p1) {
   STableObj *pTable = (STableObj *)p1;
   if (pTable->type == TSDB_SUPER_TABLE) {
@@ -703,7 +711,7 @@ static int32_t mnodeProcessCreateTableMsg(SMnodeMsg *pMsg) {
   
   if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pCreate->db);
   if (pMsg->pDb == NULL) {
-    mError("app:%p:%p, table:%s, failed to create, db not selected", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
+    mError("msg:%p, app:%p table:%s, failed to create, db not selected", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId);
     return TSDB_CODE_MND_DB_NOT_SELECTED;
   }
   
@@ -715,24 +723,24 @@ static int32_t mnodeProcessCreateTableMsg(SMnodeMsg *pMsg) {
   if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pCreate->tableId);
   if (pMsg->pTable != NULL && pMsg->retry == 0) {
     if (pCreate->getMeta) {
-      mDebug("app:%p:%p, table:%s, continue to get meta", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
+      mDebug("msg:%p, app:%p table:%s, continue to get meta", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId);
       return mnodeGetChildTableMeta(pMsg);
     } else if (pCreate->igExists) {
-      mDebug("app:%p:%p, table:%s, is already exist", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
+      mDebug("msg:%p, app:%p table:%s, is already exist", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId);
       return TSDB_CODE_SUCCESS;
     } else {
-      mError("app:%p:%p, table:%s, failed to create, table already exist", pMsg->rpcMsg.ahandle, pMsg,
+      mError("msg:%p, app:%p table:%s, failed to create, table already exist", pMsg, pMsg->rpcMsg.ahandle,
              pCreate->tableId);
       return TSDB_CODE_MND_TABLE_ALREADY_EXIST;
     }
   }
 
   if (pCreate->numOfTags != 0) {
-    mDebug("app:%p:%p, table:%s, create stable msg is received from thandle:%p", pMsg->rpcMsg.ahandle, pMsg,
+    mDebug("msg:%p, app:%p table:%s, create stable msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle,
            pCreate->tableId, pMsg->rpcMsg.handle);
     return mnodeProcessCreateSuperTableMsg(pMsg);
   } else {
-    mDebug("app:%p:%p, table:%s, create ctable msg is received from thandle:%p", pMsg->rpcMsg.ahandle, pMsg,
+    mDebug("msg:%p, app:%p table:%s, create ctable msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle,
            pCreate->tableId, pMsg->rpcMsg.handle);
     return mnodeProcessCreateChildTableMsg(pMsg);
   }
@@ -742,7 +750,8 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) {
   SCMDropTableMsg *pDrop = pMsg->rpcMsg.pCont;
   if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pDrop->tableId);
   if (pMsg->pDb == NULL) {
-    mError("app:%p:%p, table:%s, failed to drop table, db not selected or db in dropping", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId);
+    mError("msg:%p, app:%p table:%s, failed to drop table, db not selected or db in dropping", pMsg,
+           pMsg->rpcMsg.ahandle, pDrop->tableId);
     return TSDB_CODE_MND_DB_NOT_SELECTED;
   }
 
@@ -752,7 +761,7 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) {
   }
 
   if (mnodeCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) {
-    mError("app:%p:%p, table:%s, failed to drop table, in monitor database", pMsg->rpcMsg.ahandle, pMsg,
+    mError("msg:%p, app:%p table:%s, failed to drop table, in monitor database", pMsg, pMsg->rpcMsg.ahandle,
            pDrop->tableId);
     return TSDB_CODE_MND_MONITOR_DB_FORBIDDEN;
   }
@@ -760,22 +769,22 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) {
   if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pDrop->tableId);
   if (pMsg->pTable == NULL) {
     if (pDrop->igNotExists) {
-      mDebug("app:%p:%p, table:%s, table is not exist, treat as success", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId);
+      mDebug("msg:%p, app:%p table:%s is not exist, treat as success", pMsg, pMsg->rpcMsg.ahandle, pDrop->tableId);
       return TSDB_CODE_SUCCESS;
     } else {
-      mError("app:%p:%p, table:%s, failed to drop table, table not exist", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId);
+      mError("msg:%p, app:%p table:%s, failed to drop, table not exist", pMsg, pMsg->rpcMsg.ahandle, pDrop->tableId);
       return TSDB_CODE_MND_INVALID_TABLE_NAME;
     }
   }
 
   if (pMsg->pTable->type == TSDB_SUPER_TABLE) {
     SSTableObj *pSTable = (SSTableObj *)pMsg->pTable;
-    mInfo("app:%p:%p, table:%s, start to drop stable, uid:%" PRIu64 ", numOfChildTables:%d, sizeOfVgList:%d",
-          pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId, pSTable->uid, pSTable->numOfTables, (int32_t)taosHashGetSize(pSTable->vgHash));
+    mInfo("msg:%p, app:%p table:%s, start to drop stable, uid:%" PRIu64 ", numOfChildTables:%d, sizeOfVgList:%d",
+          pMsg, pMsg->rpcMsg.ahandle, pDrop->tableId, pSTable->uid, pSTable->numOfTables, (int32_t)taosHashGetSize(pSTable->vgHash));
     return mnodeProcessDropSuperTableMsg(pMsg);
   } else {
     SCTableObj *pCTable = (SCTableObj *)pMsg->pTable;
-    mInfo("app:%p:%p, table:%s, start to drop ctable, vgId:%d tid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg,
+    mInfo("msg:%p, app:%p table:%s, start to drop ctable, vgId:%d tid:%d uid:%" PRIu64, pMsg, pMsg->rpcMsg.ahandle,
           pDrop->tableId, pCTable->vgId, pCTable->tid, pCTable->uid);
     return mnodeProcessDropChildTableMsg(pMsg);
   }
@@ -784,12 +793,12 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) {
 static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) {
   STableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
   pInfo->createFlag = htons(pInfo->createFlag);
-  mDebug("app:%p:%p, table:%s, table meta msg is received from thandle:%p, createFlag:%d", pMsg->rpcMsg.ahandle, pMsg,
+  mDebug("msg:%p, app:%p table:%s, table meta msg is received from thandle:%p, createFlag:%d", pMsg, pMsg->rpcMsg.ahandle,
          pInfo->tableId, pMsg->rpcMsg.handle, pInfo->createFlag);
 
   if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pInfo->tableId);
   if (pMsg->pDb == NULL) {
-    mError("app:%p:%p, table:%s, failed to get table meta, db not selected", pMsg->rpcMsg.ahandle, pMsg,
+    mError("msg:%p, app:%p table:%s, failed to get table meta, db not selected", pMsg, pMsg->rpcMsg.ahandle,
            pInfo->tableId);
     return TSDB_CODE_MND_DB_NOT_SELECTED;
   }
@@ -802,11 +811,11 @@ static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) {
   if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pInfo->tableId);
   if (pMsg->pTable == NULL) {
     if (!pInfo->createFlag) {
-      mError("app:%p:%p, table:%s, failed to get table meta, table not exist", pMsg->rpcMsg.ahandle, pMsg,
+      mError("msg:%p, app:%p table:%s, failed to get table meta, table not exist", pMsg, pMsg->rpcMsg.ahandle,
              pInfo->tableId);
       return TSDB_CODE_MND_INVALID_TABLE_NAME;
     } else {
-      mDebug("app:%p:%p, table:%s, failed to get table meta, start auto create table ", pMsg->rpcMsg.ahandle, pMsg,
+      mDebug("msg:%p, app:%p table:%s, failed to get table meta, start auto create table ", pMsg, pMsg->rpcMsg.ahandle,
              pInfo->tableId);
       return mnodeAutoCreateChildTable(pMsg);
     }
@@ -826,9 +835,9 @@ static int32_t mnodeCreateSuperTableCb(SMnodeMsg *pMsg, int32_t code) {
   if (code == TSDB_CODE_SUCCESS) {
     mLInfo("stable:%s, is created in sdb, uid:%" PRIu64, pTable->info.tableId, pTable->uid);
   } else {
-    mError("app:%p:%p, stable:%s, failed to create in sdb, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
+    mError("msg:%p, app:%p stable:%s, failed to create in sdb, reason:%s", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId,
            tstrerror(code));
-    SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .table = tsSuperTableSdb};
+    SSdbRow desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .pTable = tsSuperTableSdb};
     sdbDeleteRow(&desc);
   }
 
@@ -841,7 +850,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
   SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont;
   SSTableObj *   pStable = calloc(1, sizeof(SSTableObj));
   if (pStable == NULL) {
-    mError("app:%p:%p, table:%s, failed to create, no enough memory", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
+    mError("msg:%p, app:%p table:%s, failed to create, no enough memory", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId);
     return TSDB_CODE_MND_OUT_OF_MEMORY;
   }
 
@@ -860,7 +869,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
   pStable->schema = (SSchema *)calloc(1, schemaSize);
   if (pStable->schema == NULL) {
     free(pStable);
-    mError("app:%p:%p, table:%s, failed to create, no schema input", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
+    mError("msg:%p, app:%p table:%s, failed to create, no schema input", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId);
     return TSDB_CODE_MND_INVALID_TABLE_NAME;
   }
   memcpy(pStable->schema, pCreate->schema, numOfCols * sizeof(SSchema));
@@ -878,20 +887,20 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
   pMsg->pTable = (STableObj *)pStable;
   mnodeIncTableRef(pMsg->pTable);
 
-  SSdbOper oper = {
+  SSdbRow row = {
     .type    = SDB_OPER_GLOBAL,
-    .table = tsSuperTableSdb,
+    .pTable  = tsSuperTableSdb,
     .pObj    = pStable,
     .rowSize = sizeof(SSTableObj) + schemaSize,
     .pMsg    = pMsg,
-    .writeCb = mnodeCreateSuperTableCb
+    .fpRsp   = mnodeCreateSuperTableCb
   };
 
-  int32_t code = sdbInsertRow(&oper);
+  int32_t code = sdbInsertRow(&row);
   if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mnodeDestroySuperTable(pStable);
     pMsg->pTable = NULL;
-    mError("app:%p:%p, table:%s, failed to create, sdb error", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
+    mError("msg:%p, app:%p table:%s, failed to create, sdb error", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId);
   }
 
   return code;
@@ -900,9 +909,9 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) {
 static int32_t mnodeDropSuperTableCb(SMnodeMsg *pMsg, int32_t code) {
   SSTableObj *pTable = (SSTableObj *)pMsg->pTable;
   if (code != TSDB_CODE_SUCCESS) {
-    mError("app:%p:%p, stable:%s, failed to drop, sdb error", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId);
+    mError("msg:%p, app:%p stable:%s, failed to drop, sdb error", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId);
   } else {
-    mLInfo("app:%p:%p, stable:%s, is dropped from sdb", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId);
+    mLInfo("msg:%p, app:%p stable:%s, is dropped from sdb", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId);
   }
 
   return code;
@@ -913,10 +922,10 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
 
   SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
    if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) {
-    SHashMutableIterator *pIter = taosHashCreateIter(pStable->vgHash);
-    while (taosHashIterNext(pIter)) {
-      int32_t *pVgId = taosHashIterGet(pIter);
+    int32_t *pVgId = taosHashIterate(pStable->vgHash, NULL);
+    while (pVgId) {
       SVgObj *pVgroup = mnodeGetVgroup(*pVgId);
+      pVgId = taosHashIterate(pStable->vgHash, pVgId);
       if (pVgroup == NULL) break;
 
       SDropSTableMsg *pDrop = rpcMallocCont(sizeof(SDropSTableMsg));
@@ -925,29 +934,30 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) {
       pDrop->uid = htobe64(pStable->uid);
       mnodeExtractTableName(pStable->info.tableId, pDrop->tableId);
 
-      mInfo("app:%p:%p, stable:%s, send drop stable msg to vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
-             pVgroup->vgId);
+      mInfo("msg:%p, app:%p stable:%s, send drop stable msg to vgId:%d", pMsg, pMsg->rpcMsg.ahandle,
+            pStable->info.tableId, pVgroup->vgId);
       SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pVgroup);
       SRpcMsg   rpcMsg = {.pCont = pDrop, .contLen = sizeof(SDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE};
       dnodeSendMsgToDnode(&epSet, &rpcMsg);
       mnodeDecVgroupRef(pVgroup);
     }
-    taosHashDestroyIter(pIter);
+
+    taosHashCancelIterate(pStable->vgHash, pVgId);
 
     mnodeDropAllChildTablesInStable(pStable);
   } 
   
-  SSdbOper oper = {
+  SSdbRow row = {
     .type    = SDB_OPER_GLOBAL,
-    .table = tsSuperTableSdb,
+    .pTable  = tsSuperTableSdb,
     .pObj    = pStable,
     .pMsg    = pMsg,
-    .writeCb = mnodeDropSuperTableCb
+    .fpRsp   = mnodeDropSuperTableCb
   };
 
-  int32_t code = sdbDeleteRow(&oper);
+  int32_t code = sdbDeleteRow(&row);
   if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
-    mError("app:%p:%p, table:%s, failed to drop, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+    mError("msg:%p, app:%p table:%s, failed to drop, reason:%s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId,
            tstrerror(code));
   }
 
@@ -967,7 +977,7 @@ static int32_t mnodeFindSuperTableTagIndex(SSTableObj *pStable, const char *tagN
 
 static int32_t mnodeAddSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) {
   SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
-  mLInfo("app:%p:%p, stable %s, add tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+  mLInfo("msg:%p, app:%p stable %s, add tag result:%s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId,
           tstrerror(code));
 
   return code;
@@ -976,20 +986,20 @@ static int32_t mnodeAddSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) {
 static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t ntags) {
   SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
   if (pStable->numOfTags + ntags > TSDB_MAX_TAGS) {
-    mError("app:%p:%p, stable:%s, add tag, too many tags", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId);
+    mError("msg:%p, app:%p stable:%s, add tag, too many tags", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId);
     return TSDB_CODE_MND_TOO_MANY_TAGS;
   }
 
   for (int32_t i = 0; i < ntags; i++) {
     if (mnodeFindSuperTableColumnIndex(pStable, schema[i].name) > 0) {
-      mError("app:%p:%p, stable:%s, add tag, column:%s already exist", pMsg->rpcMsg.ahandle, pMsg,
+      mError("msg:%p, app:%p stable:%s, add tag, column:%s already exist", pMsg, pMsg->rpcMsg.ahandle,
              pStable->info.tableId, schema[i].name);
       return TSDB_CODE_MND_TAG_ALREAY_EXIST;
     }
 
     if (mnodeFindSuperTableTagIndex(pStable, schema[i].name) > 0) {
-      mError("app:%p:%p, stable:%s, add tag, tag:%s already exist", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
-             schema[i].name);
+      mError("msg:%p, app:%p stable:%s, add tag, tag:%s already exist", pMsg, pMsg->rpcMsg.ahandle,
+             pStable->info.tableId, schema[i].name);
       return TSDB_CODE_MND_FIELD_ALREAY_EXIST;
     }
   }
@@ -1007,23 +1017,23 @@ static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t
   pStable->numOfTags += ntags;
   pStable->tversion++;
 
-  mInfo("app:%p:%p, stable %s, start to add tag %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+  mInfo("msg:%p, app:%p stable %s, start to add tag %s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId,
          schema[0].name);
 
-  SSdbOper oper = {
+  SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,
-    .table = tsSuperTableSdb,
+    .pTable = tsSuperTableSdb,
     .pObj   = pStable,
     .pMsg   = pMsg,
-    .writeCb = mnodeAddSuperTableTagCb
+    .fpRsp  = mnodeAddSuperTableTagCb
   };
 
-  return sdbUpdateRow(&oper);
+  return sdbUpdateRow(&row);
 }
 
 static int32_t mnodeDropSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) {
   SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
-  mLInfo("app:%p:%p, stable %s, drop tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+  mLInfo("msg:%p, app:%p stable %s, drop tag result:%s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId,
           tstrerror(code));
   return code;
 }
@@ -1032,7 +1042,7 @@ static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, char *tagName) {
   SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
   int32_t col = mnodeFindSuperTableTagIndex(pStable, tagName);
   if (col < 0) {
-    mError("app:%p:%p, stable:%s, drop tag, tag:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+    mError("msg:%p, app:%p stable:%s, drop tag, tag:%s not exist", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId,
            tagName);
     return TSDB_CODE_MND_TAG_NOT_EXIST;
   }
@@ -1042,22 +1052,22 @@ static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, char *tagName) {
   pStable->numOfTags--;
   pStable->tversion++;
 
-  mInfo("app:%p:%p, stable %s, start to drop tag %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tagName);
+  mInfo("msg:%p, app:%p stable %s, start to drop tag %s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, tagName);
 
-  SSdbOper oper = {
+  SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,
-    .table = tsSuperTableSdb,
+    .pTable = tsSuperTableSdb,
     .pObj   = pStable,
     .pMsg   = pMsg,
-    .writeCb = mnodeDropSuperTableTagCb
+    .fpRsp  = mnodeDropSuperTableTagCb
   };
 
-  return sdbUpdateRow(&oper);
+  return sdbUpdateRow(&row);
 }
 
 static int32_t mnodeModifySuperTableTagNameCb(SMnodeMsg *pMsg, int32_t code) {
   SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
-  mLInfo("app:%p:%p, stable %s, modify tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+  mLInfo("msg:%p, app:%p stable %s, modify tag result:%s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId,
          tstrerror(code));
   return code;
 }
@@ -1066,7 +1076,7 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, c
   SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
   int32_t col = mnodeFindSuperTableTagIndex(pStable, oldTagName);
   if (col < 0) {
-    mError("app:%p:%p, stable:%s, failed to modify table tag, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg,
+    mError("msg:%p, app:%p stable:%s, failed to modify table tag, oldName: %s, newName: %s", pMsg, pMsg->rpcMsg.ahandle,
            pStable->info.tableId, oldTagName, newTagName);
     return TSDB_CODE_MND_TAG_NOT_EXIST;
   }
@@ -1085,18 +1095,18 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, c
   SSchema *schema = (SSchema *) (pStable->schema + pStable->numOfColumns + col);
   tstrncpy(schema->name, newTagName, sizeof(schema->name));
 
-  mInfo("app:%p:%p, stable %s, start to modify tag %s to %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+  mInfo("msg:%p, app:%p stable %s, start to modify tag %s to %s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId,
          oldTagName, newTagName);
 
-  SSdbOper oper = {
+  SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,
-    .table = tsSuperTableSdb,
+    .pTable = tsSuperTableSdb,
     .pObj   = pStable,
     .pMsg   = pMsg,
-    .writeCb = mnodeModifySuperTableTagNameCb
+    .fpRsp  = mnodeModifySuperTableTagNameCb
   };
 
-  return sdbUpdateRow(&oper);
+  return sdbUpdateRow(&row);
 }
 
 static int32_t mnodeFindSuperTableColumnIndex(SSTableObj *pStable, char *colName) {
@@ -1112,7 +1122,7 @@ static int32_t mnodeFindSuperTableColumnIndex(SSTableObj *pStable, char *colName
 
 static int32_t mnodeAddSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
   SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
-  mLInfo("app:%p:%p, stable %s, add column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+  mLInfo("msg:%p, app:%p stable %s, add column result:%s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId,
           tstrerror(code));
   return code;
 }
@@ -1121,19 +1131,19 @@ static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32
   SDbObj *pDb = pMsg->pDb;
   SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
   if (ncols <= 0) {
-    mError("app:%p:%p, stable:%s, add column, ncols:%d <= 0", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, ncols);
+    mError("msg:%p, app:%p stable:%s, add column, ncols:%d <= 0", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, ncols);
     return TSDB_CODE_MND_APP_ERROR;
   }
 
   for (int32_t i = 0; i < ncols; i++) {
     if (mnodeFindSuperTableColumnIndex(pStable, schema[i].name) > 0) {
-      mError("app:%p:%p, stable:%s, add column, column:%s already exist", pMsg->rpcMsg.ahandle, pMsg,
+      mError("msg:%p, app:%p stable:%s, add column, column:%s already exist", pMsg, pMsg->rpcMsg.ahandle,
              pStable->info.tableId, schema[i].name);
       return TSDB_CODE_MND_FIELD_ALREAY_EXIST;
     }
 
     if (mnodeFindSuperTableTagIndex(pStable, schema[i].name) > 0) {
-      mError("app:%p:%p, stable:%s, add column, tag:%s already exist", pMsg->rpcMsg.ahandle, pMsg,
+      mError("msg:%p, app:%p stable:%s, add column, tag:%s already exist", pMsg, pMsg->rpcMsg.ahandle,
              pStable->info.tableId, schema[i].name);
       return TSDB_CODE_MND_TAG_ALREAY_EXIST;
     }
@@ -1160,22 +1170,22 @@ static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32
     mnodeDecAcctRef(pAcct);
   }
 
-  mInfo("app:%p:%p, stable %s, start to add column", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId);
+  mInfo("msg:%p, app:%p stable %s, start to add column", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId);
 
-  SSdbOper oper = {
+  SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,
-    .table = tsSuperTableSdb,
+    .pTable = tsSuperTableSdb,
     .pObj   = pStable,
     .pMsg   = pMsg,
-    .writeCb = mnodeAddSuperTableColumnCb
+    .fpRsp  = mnodeAddSuperTableColumnCb
   };
 
- return sdbUpdateRow(&oper);
+ return sdbUpdateRow(&row);
 }
 
 static int32_t mnodeDropSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
   SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
-  mLInfo("app:%p:%p, stable %s, delete column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+  mLInfo("msg:%p, app:%p stable %s, delete column result:%s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId,
          tstrerror(code));
   return code;
 }
@@ -1185,7 +1195,7 @@ static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) {
   SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
   int32_t col = mnodeFindSuperTableColumnIndex(pStable, colName);
   if (col <= 0) {
-    mError("app:%p:%p, stable:%s, drop column, column:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+    mError("msg:%p, app:%p stable:%s, drop column, column:%s not exist", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId,
            colName);
     return TSDB_CODE_MND_FIELD_NOT_EXIST;
   }
@@ -1205,22 +1215,22 @@ static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) {
     mnodeDecAcctRef(pAcct);
   }
 
-  mInfo("app:%p:%p, stable %s, start to delete column", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId);
+  mInfo("msg:%p, app:%p stable %s, start to delete column", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId);
 
-  SSdbOper oper = {
+  SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,
-    .table = tsSuperTableSdb,
+    .pTable = tsSuperTableSdb,
     .pObj   = pStable,
     .pMsg   = pMsg,
-    .writeCb = mnodeDropSuperTableColumnCb
+    .fpRsp  = mnodeDropSuperTableColumnCb
   };
 
-  return sdbUpdateRow(&oper);
+  return sdbUpdateRow(&row);
 }
 
 static int32_t mnodeChangeSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
   SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
-  mLInfo("app:%p:%p, stable %s, change column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+  mLInfo("msg:%p, app:%p stable %s, change column result:%s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId,
          tstrerror(code));
   return code;
 }
@@ -1229,7 +1239,7 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char
   SSTableObj *pStable = (SSTableObj *)pMsg->pTable;
   int32_t col = mnodeFindSuperTableColumnIndex(pStable, oldName);
   if (col < 0) {
-    mError("app:%p:%p, stable:%s, change column, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg,
+    mError("msg:%p, app:%p stable:%s, change column, oldName:%s, newName:%s", pMsg, pMsg->rpcMsg.ahandle,
            pStable->info.tableId, oldName, newName);
     return TSDB_CODE_MND_FIELD_NOT_EXIST;
   }
@@ -1248,18 +1258,18 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char
   SSchema *schema = (SSchema *) (pStable->schema + col);
   tstrncpy(schema->name, newName, sizeof(schema->name));
 
-  mInfo("app:%p:%p, stable %s, start to modify column %s to %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId,
+  mInfo("msg:%p, app:%p stable %s, start to modify column %s to %s", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId,
          oldName, newName);
 
-  SSdbOper oper = {
+  SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,
-    .table = tsSuperTableSdb,
+    .pTable = tsSuperTableSdb,
     .pObj   = pStable,
     .pMsg   = pMsg,
-    .writeCb = mnodeChangeSuperTableColumnCb
+    .fpRsp  = mnodeChangeSuperTableColumnCb
   };
 
-  return sdbUpdateRow(&oper);
+  return sdbUpdateRow(&row);
 }
 
 // show super tables
@@ -1417,20 +1427,18 @@ void mnodeDropAllSuperTables(SDbObj *pDropDb) {
     if (pTable == NULL) break;
 
     if (strncmp(prefix, pTable->info.tableId, prefixLen) == 0) {
-      SSdbOper oper = {
+      SSdbRow row = {
         .type   = SDB_OPER_LOCAL,
-        .table = tsSuperTableSdb,
+        .pTable = tsSuperTableSdb,
         .pObj   = pTable,
       };
-      sdbDeleteRow(&oper);
+      sdbDeleteRow(&row);
       numOfTables ++;
     }
 
     mnodeDecTableRef(pTable);
   }
 
-  sdbFreeIter(pIter);
-
   mInfo("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables);
 }
 
@@ -1470,7 +1478,7 @@ static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) {
 
   pMsg->rpcRsp.rsp = pMeta;
 
-  mDebug("app:%p:%p, stable:%s, uid:%" PRIu64 " table meta is retrieved", pMsg->rpcMsg.ahandle, pMsg,
+  mDebug("msg:%p, app:%p stable:%s, uid:%" PRIu64 " table meta is retrieved", pMsg, pMsg->rpcMsg.ahandle,
          pTable->info.tableId, pTable->uid);
   return TSDB_CODE_SUCCESS;
 }
@@ -1503,12 +1511,12 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
     char *          stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i;
     SSTableObj *pTable = mnodeGetSuperTable(stableName);
     if (pTable == NULL) {
-      mError("app:%p:%p, stable:%s, not exist while get stable vgroup info", pMsg->rpcMsg.ahandle, pMsg, stableName);
+      mError("msg:%p, app:%p stable:%s, not exist while get stable vgroup info", pMsg, pMsg->rpcMsg.ahandle, stableName);
       mnodeDecTableRef(pTable);
       continue;
     }
     if (pTable->vgHash == NULL) {
-      mDebug("app:%p:%p, stable:%s, no vgroup exist while get stable vgroup info", pMsg->rpcMsg.ahandle, pMsg,
+      mDebug("msg:%p, app:%p stable:%s, no vgroup exist while get stable vgroup info", pMsg, pMsg->rpcMsg.ahandle,
              stableName);
       mnodeDecTableRef(pTable);
 
@@ -1522,11 +1530,11 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
     } else {
       SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)msg;
 
-      SHashMutableIterator *pIter = taosHashCreateIter(pTable->vgHash);
+      int32_t *pVgId = taosHashIterate(pTable->vgHash, NULL);
       int32_t  vgSize = 0;
-      while (taosHashIterNext(pIter)) {
-        int32_t *pVgId = taosHashIterGet(pIter);
-        SVgObj * pVgroup = mnodeGetVgroup(*pVgId);
+      while (pVgId) {
+        SVgObj *pVgroup = mnodeGetVgroup(*pVgId);
+        pVgId = taosHashIterate(pTable->vgHash, pVgId);
         if (pVgroup == NULL) continue;
 
         pVgroupMsg->vgroups[vgSize].vgId = htonl(pVgroup->vgId);
@@ -1546,7 +1554,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
         mnodeDecVgroupRef(pVgroup);
       }
 
-      taosHashDestroyIter(pIter);
+      taosHashCancelIterate(pTable->vgHash, pVgId);
       mnodeDecTableRef(pTable);
 
       pVgroupMsg->numOfVgroups = htonl(vgSize);
@@ -1650,7 +1658,7 @@ static int32_t mnodeDoCreateChildTableFp(SMnodeMsg *pMsg) {
   SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
   assert(pTable);
 
-  mDebug("app:%p:%p, table:%s, created in mnode, vgId:%d sid:%d, uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg,
+  mDebug("msg:%p, app:%p table:%s, created in mnode, vgId:%d sid:%d, uid:%" PRIu64, pMsg, pMsg->rpcMsg.ahandle,
          pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid);
 
   SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont;
@@ -1679,22 +1687,22 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) {
 
   if (code == TSDB_CODE_SUCCESS) {
     if (pCreate->getMeta) {
-      mDebug("app:%p:%p, table:%s, created in dnode and continue to get meta, thandle:%p", pMsg->rpcMsg.ahandle, pMsg,
-             pTable->info.tableId, pMsg->rpcMsg.handle);
+      mDebug("msg:%p, app:%p table:%s, created in dnode and continue to get meta, thandle:%p", pMsg,
+             pMsg->rpcMsg.ahandle, pTable->info.tableId, pMsg->rpcMsg.handle);
 
       pMsg->retry = 0;
       dnodeReprocessMWriteMsg(pMsg);
     } else {
-      mDebug("app:%p:%p, table:%s, created in dnode, thandle:%p", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
+      mDebug("msg:%p, app:%p table:%s, created in dnode, thandle:%p", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId,
              pMsg->rpcMsg.handle);
 
       dnodeSendRpcMWriteRsp(pMsg, TSDB_CODE_SUCCESS);
     }
     return TSDB_CODE_MND_ACTION_IN_PROGRESS;
   } else {
-    mError("app:%p:%p, table:%s, failed to create table sid:%d, uid:%" PRIu64 ", reason:%s", pMsg->rpcMsg.ahandle, pMsg,
-           pTable->info.tableId, pTable->tid, pTable->uid, tstrerror(code));
-    SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .table = tsChildTableSdb};
+    mError("msg:%p, app:%p table:%s, failed to create table sid:%d, uid:%" PRIu64 ", reason:%s", pMsg,
+           pMsg->rpcMsg.ahandle, pTable->info.tableId, pTable->tid, pTable->uid, tstrerror(code));
+    SSdbRow desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .pTable = tsChildTableSdb};
     sdbDeleteRow(&desc);
     return code;
   }
@@ -1705,7 +1713,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) {
   SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont;
   SCTableObj *pTable = calloc(1, sizeof(SCTableObj));
   if (pTable == NULL) {
-    mError("app:%p:%p, table:%s, failed to alloc memory", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
+    mError("msg:%p, app:%p table:%s, failed to alloc memory", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId);
     return TSDB_CODE_MND_OUT_OF_MEMORY;
   }
 
@@ -1724,7 +1732,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) {
     STagData *pTagData = (STagData *)pCreate->schema;  // it is a tag key
     if (pMsg->pSTable == NULL) pMsg->pSTable = mnodeGetSuperTable(pTagData->name);
     if (pMsg->pSTable == NULL) {
-      mError("app:%p:%p, table:%s, corresponding super table:%s does not exist", pMsg->rpcMsg.ahandle, pMsg,
+      mError("msg:%p, app:%p table:%s, corresponding super table:%s does not exist", pMsg, pMsg->rpcMsg.ahandle,
              pCreate->tableId, pTagData->name);
       mnodeDestroyChildTable(pTable);
       return TSDB_CODE_MND_INVALID_TABLE_NAME;
@@ -1772,7 +1780,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) {
       }
       memcpy(pTable->sql, (char *) (pCreate->schema) + numOfCols * sizeof(SSchema), pTable->sqlLen);
       pTable->sql[pTable->sqlLen - 1] = 0;
-      mDebug("app:%p:%p, table:%s, stream sql len:%d sql:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
+      mDebug("msg:%p, app:%p table:%s, stream sql len:%d sql:%s", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId,
              pTable->sqlLen, pTable->sql);
     }
   }
@@ -1780,22 +1788,22 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) {
   pMsg->pTable = (STableObj *)pTable;
   mnodeIncTableRef(pMsg->pTable);
 
-  SSdbOper desc = {
+  SSdbRow desc = {
     .type   = SDB_OPER_GLOBAL,
     .pObj   = pTable,
-    .table = tsChildTableSdb,
+    .pTable = tsChildTableSdb,
     .pMsg   = pMsg,
-    .reqFp = mnodeDoCreateChildTableFp
+    .fpReq  = mnodeDoCreateChildTableFp
   };
   
   int32_t code = sdbInsertRow(&desc);
   if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mnodeDestroyChildTable(pTable);
     pMsg->pTable = NULL;
-    mError("app:%p:%p, table:%s, failed to create, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId,
+    mError("msg:%p, app:%p table:%s, failed to create, reason:%s", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId,
            tstrerror(code));
   } else {
-    mDebug("app:%p:%p, table:%s, allocated in vgroup, vgId:%d sid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg,
+    mDebug("msg:%p, app:%p table:%s, allocated in vgroup, vgId:%d sid:%d uid:%" PRIu64, pMsg, pMsg->rpcMsg.ahandle,
            pTable->info.tableId, pVgroup->vgId, pTable->tid, pTable->uid);
   }
 
@@ -1806,7 +1814,7 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) {
   SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont;
   int32_t code = grantCheck(TSDB_GRANT_TIMESERIES);
   if (code != TSDB_CODE_SUCCESS) {
-    mError("app:%p:%p, table:%s, failed to create, grant timeseries failed", pMsg->rpcMsg.ahandle, pMsg,
+    mError("msg:%p, app:%p table:%s, failed to create, grant timeseries failed", pMsg, pMsg->rpcMsg.ahandle,
            pCreate->tableId);
     return code;
   }
@@ -1817,7 +1825,7 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) {
       int32_t tid = 0;
       code = mnodeGetAvailableVgroup(pMsg, &pVgroup, &tid);
       if (code != TSDB_CODE_SUCCESS) {
-        mDebug("app:%p:%p, table:%s, failed to get available vgroup, reason:%s", pMsg->rpcMsg.ahandle, pMsg,
+        mDebug("msg:%p, app:%p table:%s, failed to get available vgroup, reason:%s", pMsg, pMsg->rpcMsg.ahandle,
                pCreate->tableId, tstrerror(code));
         return code;
       }
@@ -1836,22 +1844,22 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) {
   }
 
   if (pMsg->pTable == NULL) {
-    mError("app:%p:%p, table:%s, object not found, retry:%d reason:%s", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId, pMsg->retry,
+    mError("msg:%p, app:%p table:%s, object not found, retry:%d reason:%s", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId, pMsg->retry,
            tstrerror(terrno));
     return terrno;
   } else {
-    mDebug("app:%p:%p, table:%s, send create msg to vnode again", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId);
+    mDebug("msg:%p, app:%p table:%s, send create msg to vnode again", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId);
     return mnodeDoCreateChildTableFp(pMsg);
   }
 }
 
 static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) {
   SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
-  mLInfo("app:%p:%p, ctable:%s, is dropped from sdb", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId);
+  mLInfo("msg:%p, app:%p ctable:%s, is dropped from sdb", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId);
 
   SMDDropTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropTableMsg));
   if (pDrop == NULL) {
-    mError("app:%p:%p, ctable:%s, failed to drop ctable, no enough memory", pMsg->rpcMsg.ahandle, pMsg,
+    mError("msg:%p, app:%p ctable:%s, failed to drop ctable, no enough memory", pMsg, pMsg->rpcMsg.ahandle,
            pTable->info.tableId);
     return TSDB_CODE_MND_OUT_OF_MEMORY;
   }
@@ -1864,7 +1872,7 @@ static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) {
 
   SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pMsg->pVgroup);
 
-  mInfo("app:%p:%p, ctable:%s, send drop ctable msg, vgId:%d sid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg,
+  mInfo("msg:%p, app:%p ctable:%s, send drop ctable msg, vgId:%d sid:%d uid:%" PRIu64, pMsg, pMsg->rpcMsg.ahandle,
         pDrop->tableId, pTable->vgId, pTable->tid, pTable->uid);
 
   SRpcMsg rpcMsg = {
@@ -1885,7 +1893,7 @@ static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) {
 static int32_t mnodeDropChildTableCb(SMnodeMsg *pMsg, int32_t code) {
   if (code != TSDB_CODE_SUCCESS) {
     SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
-    mError("app:%p:%p, ctable:%s, failed to drop, sdb error", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId);
+    mError("msg:%p, app:%p ctable:%s, failed to drop, sdb error", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId);
     return code;
   } 
 
@@ -1896,22 +1904,22 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) {
   SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
   if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId);
   if (pMsg->pVgroup == NULL) {
-    mError("app:%p:%p, table:%s, failed to drop ctable, vgroup not exist", pMsg->rpcMsg.ahandle, pMsg,
+    mError("msg:%p, app:%p table:%s, failed to drop ctable, vgroup not exist", pMsg, pMsg->rpcMsg.ahandle,
            pTable->info.tableId);
     return TSDB_CODE_MND_APP_ERROR;
   }
 
-  SSdbOper oper = {
+  SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,
-    .table = tsChildTableSdb,
+    .pTable = tsChildTableSdb,
     .pObj   = pTable,
     .pMsg   = pMsg,
-    .writeCb = mnodeDropChildTableCb
+    .fpRsp  = mnodeDropChildTableCb
   };
 
-  int32_t code = sdbDeleteRow(&oper);
+  int32_t code = sdbDeleteRow(&row);
   if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
-    mError("app:%p:%p, ctable:%s, failed to drop, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
+    mError("msg:%p, app:%p ctable:%s, failed to drop, reason:%s", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId,
            tstrerror(code));
   }
 
@@ -1932,7 +1940,7 @@ static int32_t mnodeFindNormalTableColumnIndex(SCTableObj *pTable, char *colName
 static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
   SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
   if (code != TSDB_CODE_SUCCESS) {
-    mError("app:%p:%p, ctable %s, failed to alter column, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
+    mError("msg:%p, app:%p ctable %s, failed to alter column, reason:%s", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId,
            tstrerror(code));
     return code;
   }
@@ -1946,7 +1954,7 @@ static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
     pMsg->pVgroup = mnodeGetVgroup(pTable->vgId);
     if (pMsg->pVgroup == NULL) {
       rpcFreeCont(pMDCreate);
-      mError("app:%p:%p, ctable %s, vgId:%d not exist in mnode", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
+      mError("msg:%p, app:%p ctable %s, vgId:%d not exist in mnode", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId,
              pTable->vgId);
       return TSDB_CODE_MND_VGROUP_NOT_EXIST;
     }
@@ -1961,7 +1969,7 @@ static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) {
       .msgType = TSDB_MSG_TYPE_MD_ALTER_TABLE
   };
 
-  mDebug("app:%p:%p, ctable %s, send alter column msg to vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
+  mDebug("msg:%p, app:%p ctable %s, send alter column msg to vgId:%d", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId,
          pMsg->pVgroup->vgId);
 
   dnodeSendMsgToDnode(&epSet, &rpcMsg);
@@ -1972,13 +1980,13 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int3
   SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
   SDbObj *pDb = pMsg->pDb;
   if (ncols <= 0) {
-    mError("app:%p:%p, ctable:%s, add column, ncols:%d <= 0", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, ncols);
+    mError("msg:%p, app:%p ctable:%s, add column, ncols:%d <= 0", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, ncols);
     return TSDB_CODE_MND_APP_ERROR;
   }
 
   for (int32_t i = 0; i < ncols; i++) {
     if (mnodeFindNormalTableColumnIndex(pTable, schema[i].name) > 0) {
-      mError("app:%p:%p, ctable:%s, add column, column:%s already exist", pMsg->rpcMsg.ahandle, pMsg,
+      mError("msg:%p, app:%p ctable:%s, add column, column:%s already exist", pMsg, pMsg->rpcMsg.ahandle,
              pTable->info.tableId, schema[i].name);
       return TSDB_CODE_MND_FIELD_ALREAY_EXIST;
     }
@@ -2003,17 +2011,17 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int3
     mnodeDecAcctRef(pAcct);
   }
 
-  mInfo("app:%p:%p, ctable %s, start to add column", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId);
+  mInfo("msg:%p, app:%p ctable %s, start to add column", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId);
 
-  SSdbOper oper = {
+  SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,
-    .table = tsChildTableSdb,
+    .pTable = tsChildTableSdb,
     .pObj   = pTable,
     .pMsg   = pMsg,
-    .writeCb = mnodeAlterNormalTableColumnCb
+    .fpRsp  = mnodeAlterNormalTableColumnCb
   };
 
-  return sdbUpdateRow(&oper);
+  return sdbUpdateRow(&row);
 }
 
 static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) {
@@ -2021,8 +2029,8 @@ static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) {
   SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
   int32_t col = mnodeFindNormalTableColumnIndex(pTable, colName);
   if (col <= 0) {
-    mError("app:%p:%p, ctable:%s, drop column, column:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
-           colName);
+    mError("msg:%p, app:%p ctable:%s, drop column, column:%s not exist", pMsg, pMsg->rpcMsg.ahandle,
+           pTable->info.tableId, colName);
     return TSDB_CODE_MND_FIELD_NOT_EXIST;
   }
 
@@ -2036,24 +2044,24 @@ static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) {
     mnodeDecAcctRef(pAcct);
   }
 
-  mInfo("app:%p:%p, ctable %s, start to drop column %s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, colName);
+  mInfo("msg:%p, app:%p ctable %s, start to drop column %s", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, colName);
 
-  SSdbOper oper = {
+  SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,
-    .table = tsChildTableSdb,
+    .pTable = tsChildTableSdb,
     .pObj   = pTable,
     .pMsg   = pMsg,
-    .writeCb = mnodeAlterNormalTableColumnCb
+    .fpRsp  = mnodeAlterNormalTableColumnCb
   };
 
-  return sdbUpdateRow(&oper);
+  return sdbUpdateRow(&row);
 }
 
 static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char *newName) {
   SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
   int32_t col = mnodeFindNormalTableColumnIndex(pTable, oldName);
   if (col < 0) {
-    mError("app:%p:%p, ctable:%s, change column, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg,
+    mError("msg:%p, app:%p ctable:%s, change column, oldName: %s, newName: %s", pMsg, pMsg->rpcMsg.ahandle,
            pTable->info.tableId, oldName, newName);
     return TSDB_CODE_MND_FIELD_NOT_EXIST;
   }
@@ -2072,18 +2080,18 @@ static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char
   SSchema *schema = (SSchema *) (pTable->schema + col);
   tstrncpy(schema->name, newName, sizeof(schema->name));
 
-  mInfo("app:%p:%p, ctable %s, start to modify column %s to %s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId,
+  mInfo("msg:%p, app:%p ctable %s, start to modify column %s to %s", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId,
          oldName, newName);
 
-  SSdbOper oper = {
+  SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,
-    .table = tsChildTableSdb,
+    .pTable = tsChildTableSdb,
     .pObj   = pTable,
     .pMsg   = pMsg,
-    .writeCb = mnodeAlterNormalTableColumnCb
+    .fpRsp  = mnodeAlterNormalTableColumnCb
   };
 
-  return sdbUpdateRow(&oper);
+  return sdbUpdateRow(&row);
 }
 
 static int32_t mnodeSetSchemaFromNormalTable(SSchema *pSchema, SCTableObj *pTable) {
@@ -2128,7 +2136,7 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) {
 
   if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId);
   if (pMsg->pVgroup == NULL) {
-    mError("app:%p:%p, table:%s, failed to get table meta, vgroup not exist", pMsg->rpcMsg.ahandle, pMsg,
+    mError("msg:%p, app:%p table:%s, failed to get table meta, vgroup not exist", pMsg, pMsg->rpcMsg.ahandle,
            pTable->info.tableId);
     return TSDB_CODE_MND_VGROUP_NOT_EXIST;
   }
@@ -2143,7 +2151,7 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) {
   }
   pMeta->vgroup.vgId = htonl(pMsg->pVgroup->vgId);
 
-  mDebug("app:%p:%p, table:%s, uid:%" PRIu64 " table meta is retrieved, vgId:%d sid:%d", pMsg->rpcMsg.ahandle, pMsg,
+  mDebug("msg:%p, app:%p table:%s, uid:%" PRIu64 " table meta is retrieved, vgId:%d sid:%d", pMsg, pMsg->rpcMsg.ahandle,
          pTable->info.tableId, pTable->uid, pTable->vgId, pTable->tid);
 
   return TSDB_CODE_SUCCESS;
@@ -2154,16 +2162,16 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) {
   STagData *pTags = (STagData *)pInfo->tags;
   int32_t tagLen = htonl(pTags->dataLen);
   if (pTags->name[0] == 0) {
-    mError("app:%p:%p, table:%s, failed to create table on demand for stable is empty, tagLen:%d", pMsg->rpcMsg.ahandle,
-           pMsg, pInfo->tableId, tagLen);
+    mError("msg:%p, app:%p table:%s, failed to create table on demand for stable is empty, tagLen:%d", pMsg,
+           pMsg->rpcMsg.ahandle, pInfo->tableId, tagLen);
     return TSDB_CODE_MND_INVALID_STABLE_NAME;
   }
 
   int32_t contLen = sizeof(SCMCreateTableMsg) + offsetof(STagData, data) + tagLen;
   SCMCreateTableMsg *pCreateMsg = calloc(1, contLen);
   if (pCreateMsg == NULL) {
-    mError("app:%p:%p, table:%s, failed to create table while get meta info, no enough memory", pMsg->rpcMsg.ahandle,
-           pMsg, pInfo->tableId);
+    mError("msg:%p, app:%p table:%s, failed to create table while get meta info, no enough memory", pMsg,
+           pMsg->rpcMsg.ahandle, pInfo->tableId);
     return TSDB_CODE_MND_OUT_OF_MEMORY;
   }
 
@@ -2175,8 +2183,8 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) {
   pCreateMsg->contLen = htonl(contLen);
 
   memcpy(pCreateMsg->schema, pTags, contLen - sizeof(SCMCreateTableMsg));
-  mDebug("app:%p:%p, table:%s, start to create on demand, tagLen:%d stable:%s",
-         pMsg->rpcMsg.ahandle, pMsg, pInfo->tableId, tagLen, pTags->name);
+  mDebug("msg:%p, app:%p table:%s, start to create on demand, tagLen:%d stable:%s", pMsg, pMsg->rpcMsg.ahandle,
+         pInfo->tableId, tagLen, pTags->name);
 
   if (pMsg->rpcMsg.pCont != pMsg->pCont) {
     tfree(pMsg->rpcMsg.pCont);
@@ -2192,7 +2200,7 @@ static int32_t mnodeGetChildTableMeta(SMnodeMsg *pMsg) {
   STableMetaMsg *pMeta =
       rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16));
   if (pMeta == NULL) {
-    mError("app:%p:%p, table:%s, failed to get table meta, no enough memory", pMsg->rpcMsg.ahandle, pMsg,
+    mError("msg:%p, app:%p table:%s, failed to get table meta, no enough memory", pMsg, pMsg->rpcMsg.ahandle,
            pMsg->pTable->tableId);
     return TSDB_CODE_MND_OUT_OF_MEMORY;
   }
@@ -2218,19 +2226,17 @@ void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup) {
     if (pTable == NULL) break;
 
     if (pTable->vgId == pVgroup->vgId) {
-      SSdbOper oper = {
+      SSdbRow row = {
         .type   = SDB_OPER_LOCAL,
-        .table = tsChildTableSdb,
+        .pTable = tsChildTableSdb,
         .pObj   = pTable,
       };
-      sdbDeleteRow(&oper);
+      sdbDeleteRow(&row);
       numOfTables++;
     }
     mnodeDecTableRef(pTable);
   }
 
-  sdbFreeIter(pIter);
-
   mInfo("vgId:%d, all child tables is dropped from sdb", pVgroup->vgId);
 }
 
@@ -2251,19 +2257,17 @@ void mnodeDropAllChildTables(SDbObj *pDropDb) {
     if (pTable == NULL) break;
 
     if (strncmp(prefix, pTable->info.tableId, prefixLen) == 0) {
-      SSdbOper oper = {
+      SSdbRow row = {
         .type   = SDB_OPER_LOCAL,
-        .table = tsChildTableSdb,
+        .pTable = tsChildTableSdb,
         .pObj   = pTable,
       };
-      sdbDeleteRow(&oper);
+      sdbDeleteRow(&row);
       numOfTables++;
     }
     mnodeDecTableRef(pTable);
   }
 
-  sdbFreeIter(pIter);
-
   mInfo("db:%s, all child tables:%d is dropped from sdb", pDropDb->name, numOfTables);
 }
 
@@ -2280,20 +2284,18 @@ static void mnodeDropAllChildTablesInStable(SSTableObj *pStable) {
     if (pTable == NULL) break;
 
     if (pTable->superTable == pStable) {
-      SSdbOper oper = {
+      SSdbRow row = {
         .type   = SDB_OPER_LOCAL,
-        .table = tsChildTableSdb,
+        .pTable = tsChildTableSdb,
         .pObj   = pTable,
       };
-      sdbDeleteRow(&oper);
+      sdbDeleteRow(&row);
       numOfTables++;
     }
 
     mnodeDecTableRef(pTable);
   }
 
-  sdbFreeIter(pIter);
-
   mInfo("stable:%s, all child tables:%d is dropped from sdb", pStable->info.tableId, numOfTables);
 }
 
@@ -2317,12 +2319,12 @@ static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) {
   pCfg->dnodeId = htonl(pCfg->dnodeId);
   pCfg->vgId = htonl(pCfg->vgId);
   pCfg->sid = htonl(pCfg->sid);
-  mDebug("app:%p:%p, dnode:%d, vgId:%d sid:%d, receive table config msg", pMsg->rpcMsg.ahandle, pMsg, pCfg->dnodeId,
+  mDebug("msg:%p, app:%p dnode:%d, vgId:%d sid:%d, receive table config msg", pMsg, pMsg->rpcMsg.ahandle, pCfg->dnodeId,
          pCfg->vgId, pCfg->sid);
 
   SCTableObj *pTable = mnodeGetTableByPos(pCfg->vgId, pCfg->sid);
   if (pTable == NULL) {
-    mError("app:%p:%p, dnode:%d, vgId:%d sid:%d, table not found", pMsg->rpcMsg.ahandle, pMsg, pCfg->dnodeId,
+    mError("msg:%p, app:%p dnode:%d, vgId:%d sid:%d, table not found", pMsg, pMsg->rpcMsg.ahandle, pCfg->dnodeId,
            pCfg->vgId, pCfg->sid);
     return TSDB_CODE_MND_INVALID_TABLE_ID;
   }
@@ -2343,38 +2345,37 @@ static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) {
 static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) {
   if (rpcMsg->ahandle == NULL) return;
 
-  SMnodeMsg *mnodeMsg = rpcMsg->ahandle;
-  mnodeMsg->received++;
+  SMnodeMsg *pMsg = rpcMsg->ahandle;
+  pMsg->received++;
 
-  SCTableObj *pTable = (SCTableObj *)mnodeMsg->pTable;
+  SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
   assert(pTable);
 
-  mInfo("app:%p:%p, table:%s, drop table rsp received, vgId:%d sid:%d uid:%" PRIu64 ", thandle:%p result:%s",
-        mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid,
-        mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code));
+  mInfo("msg:%p, app:%p table:%s, drop table rsp received, vgId:%d sid:%d uid:%" PRIu64 ", thandle:%p result:%s", pMsg,
+        pMsg->rpcMsg.ahandle, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid, pMsg->rpcMsg.handle,
+        tstrerror(rpcMsg->code));
 
   if (rpcMsg->code != TSDB_CODE_SUCCESS) {
-    mError("app:%p:%p, table:%s, failed to drop in dnode, vgId:%d sid:%d uid:%" PRIu64 ", reason:%s",
-           mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid,
-           tstrerror(rpcMsg->code));
-    dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code);
+    mError("msg:%p, app:%p table:%s, failed to drop in dnode, vgId:%d sid:%d uid:%" PRIu64 ", reason:%s", pMsg,
+           pMsg->rpcMsg.ahandle, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid, tstrerror(rpcMsg->code));
+    dnodeSendRpcMWriteRsp(pMsg, rpcMsg->code);
     return;
   }
 
-  if (mnodeMsg->pVgroup == NULL) mnodeMsg->pVgroup = mnodeGetVgroup(pTable->vgId);
-  if (mnodeMsg->pVgroup == NULL) {
-    mError("app:%p:%p, table:%s, failed to get vgroup", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId);
-    dnodeSendRpcMWriteRsp(mnodeMsg, TSDB_CODE_MND_VGROUP_NOT_EXIST);
+  if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId);
+  if (pMsg->pVgroup == NULL) {
+    mError("msg:%p, app:%p table:%s, failed to get vgroup", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId);
+    dnodeSendRpcMWriteRsp(pMsg, TSDB_CODE_MND_VGROUP_NOT_EXIST);
     return;
   }
 
-  if (mnodeMsg->pVgroup->numOfTables <= 0) {
-    mInfo("app:%p:%p, vgId:%d, all tables is dropped, drop vgroup", mnodeMsg->rpcMsg.ahandle, mnodeMsg,
-           mnodeMsg->pVgroup->vgId);
-    mnodeDropVgroup(mnodeMsg->pVgroup, NULL);
+  if (pMsg->pVgroup->numOfTables <= 0) {
+    mInfo("msg:%p, app:%p vgId:%d, all tables is dropped, drop vgroup", pMsg, pMsg->rpcMsg.ahandle,
+          pMsg->pVgroup->vgId);
+    mnodeDropVgroup(pMsg->pVgroup, NULL);
   }
 
-  dnodeSendRpcMWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS);
+  dnodeSendRpcMWriteRsp(pMsg, TSDB_CODE_SUCCESS);
 }
 
 /*
@@ -2384,70 +2385,70 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) {
 static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
   if (rpcMsg->ahandle == NULL) return;
 
-  SMnodeMsg *mnodeMsg = rpcMsg->ahandle;
-  mnodeMsg->received++;
+  SMnodeMsg *pMsg = rpcMsg->ahandle;
+  pMsg->received++;
 
-  SCTableObj *pTable = (SCTableObj *)mnodeMsg->pTable;
+  SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
   assert(pTable);
 
   // If the table is deleted by another thread during creation, stop creating and send drop msg to vnode
   if (sdbCheckRowDeleted(tsChildTableSdb, pTable)) {
-    mDebug("app:%p:%p, table:%s, create table rsp received, but a deleting opertion incoming, vgId:%d sid:%d uid:%" PRIu64,
-           mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid);
+    mDebug("msg:%p, app:%p table:%s, create table rsp received, but a deleting opertion incoming, vgId:%d sid:%d uid:%" PRIu64,
+           pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid);
 
     // if the vgroup is already dropped from hash, it can't be accquired by pTable->vgId
     // so the refCount of vgroup can not be decreased
     // SVgObj *pVgroup = mnodeGetVgroup(pTable->vgId);
     // if (pVgroup == NULL) {
-    //   mnodeRemoveTableFromVgroup(mnodeMsg->pVgroup, pTable);
+    //   mnodeRemoveTableFromVgroup(pMsg->pVgroup, pTable);
     // }
     // mnodeDecVgroupRef(pVgroup);
 
-    mnodeSendDropChildTableMsg(mnodeMsg, false);
+    mnodeSendDropChildTableMsg(pMsg, false);
     rpcMsg->code = TSDB_CODE_SUCCESS;
-    dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code);
+    dnodeSendRpcMWriteRsp(pMsg, rpcMsg->code);
     return;
   }
 
   if (rpcMsg->code == TSDB_CODE_SUCCESS || rpcMsg->code == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) {
-     SSdbOper desc = {
+     SSdbRow desc = {
       .type   = SDB_OPER_GLOBAL,
       .pObj   = pTable,
-      .table = tsChildTableSdb,
-      .pMsg  = mnodeMsg,
-      .writeCb = mnodeDoCreateChildTableCb
+      .pTable = tsChildTableSdb,
+      .pMsg   = pMsg,
+      .fpRsp  = mnodeDoCreateChildTableCb
     };
     
-    int32_t code = sdbInsertRowImp(&desc);
+    int32_t code = sdbInsertRowToQueue(&desc);
     if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
-      mnodeMsg->pTable = NULL;
+      pMsg->pTable = NULL;
       mnodeDestroyChildTable(pTable);
-      dnodeSendRpcMWriteRsp(mnodeMsg, code);
+      dnodeSendRpcMWriteRsp(pMsg, code);
     }
   } else {
-    mnodeMsg->retry++;
+    pMsg->retry++;
     int32_t sec = taosGetTimestampSec();
-    if (mnodeMsg->retry < CREATE_CTABLE_RETRY_TIMES && ABS(sec - mnodeMsg->incomingTs) < CREATE_CTABLE_RETRY_SEC) {
-      mDebug("app:%p:%p, table:%s, create table rsp received, need retry, times:%d vgId:%d sid:%d uid:%" PRIu64
+    if (pMsg->retry < CREATE_CTABLE_RETRY_TIMES && ABS(sec - pMsg->incomingTs) < CREATE_CTABLE_RETRY_SEC) {
+      mDebug("msg:%p, app:%p table:%s, create table rsp received, need retry, times:%d vgId:%d sid:%d uid:%" PRIu64
              " result:%s thandle:%p",
-             mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, pTable->vgId, pTable->tid,
-             pTable->uid, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle);
+             pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, pMsg->retry, pTable->vgId, pTable->tid, pTable->uid,
+             tstrerror(rpcMsg->code), pMsg->rpcMsg.handle);
 
-      dnodeDelayReprocessMWriteMsg(mnodeMsg);
+      dnodeDelayReprocessMWriteMsg(pMsg);
     } else {
-      mError("app:%p:%p, table:%s, failed to create in dnode, vgId:%d sid:%d uid:%" PRIu64
+      mError("msg:%p, app:%p table:%s, failed to create in dnode, vgId:%d sid:%d uid:%" PRIu64
              ", result:%s thandle:%p incomingTs:%d curTs:%d retryTimes:%d",
-             mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid,
-             tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle, mnodeMsg->incomingTs, sec, mnodeMsg->retry);
+             pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid,
+             tstrerror(rpcMsg->code), pMsg->rpcMsg.handle, pMsg->incomingTs, sec, pMsg->retry);
 
-      SSdbOper oper = {.type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable};
-      sdbDeleteRow(&oper);
+      SSdbRow row = {.type = SDB_OPER_GLOBAL, .pTable = tsChildTableSdb, .pObj = pTable};
+      sdbDeleteRow(&row);
 
       if (rpcMsg->code == TSDB_CODE_APP_NOT_READY) {
         //Avoid retry again in client
         rpcMsg->code = TSDB_CODE_MND_VGROUP_NOT_READY;
       }
-      dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code);
+      dnodeSendRpcMWriteRsp(pMsg, rpcMsg->code);
     }
   }
 }
@@ -2455,28 +2456,28 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) {
 static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg) {
   if (rpcMsg->ahandle == NULL) return;
 
-  SMnodeMsg *mnodeMsg = rpcMsg->ahandle;
-  mnodeMsg->received++;
+  SMnodeMsg *pMsg = rpcMsg->ahandle;
+  pMsg->received++;
 
-  SCTableObj *pTable = (SCTableObj *)mnodeMsg->pTable;
+  SCTableObj *pTable = (SCTableObj *)pMsg->pTable;
   assert(pTable);
 
   if (rpcMsg->code == TSDB_CODE_SUCCESS || rpcMsg->code == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) {
-    mDebug("app:%p:%p, ctable:%s, altered in dnode, thandle:%p result:%s", mnodeMsg->rpcMsg.ahandle, mnodeMsg,
-           pTable->info.tableId, mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code));
+    mDebug("msg:%p, app:%p ctable:%s, altered in dnode, thandle:%p result:%s", pMsg, pMsg->rpcMsg.ahandle,
+           pTable->info.tableId, pMsg->rpcMsg.handle, tstrerror(rpcMsg->code));
 
-    dnodeSendRpcMWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS);
+    dnodeSendRpcMWriteRsp(pMsg, TSDB_CODE_SUCCESS);
   } else {
-    if (mnodeMsg->retry++ < ALTER_CTABLE_RETRY_TIMES) {
-      mDebug("app:%p:%p, table:%s, alter table rsp received, need retry, times:%d result:%s thandle:%p",
-             mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, tstrerror(rpcMsg->code),
-             mnodeMsg->rpcMsg.handle);
+    if (pMsg->retry++ < ALTER_CTABLE_RETRY_TIMES) {
+      mDebug("msg:%p, app:%p table:%s, alter table rsp received, need retry, times:%d result:%s thandle:%p",
+             pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, pMsg->retry, tstrerror(rpcMsg->code),
+             pMsg->rpcMsg.handle);
 
-      dnodeDelayReprocessMWriteMsg(mnodeMsg);
+      dnodeDelayReprocessMWriteMsg(pMsg);
     } else {
-      mError("app:%p:%p, table:%s, failed to alter in dnode, result:%s thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg,
-             pTable->info.tableId, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle);
-      dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code);
+      mError("msg:%p, app:%p table:%s, failed to alter in dnode, result:%s thandle:%p", pMsg, pMsg->rpcMsg.ahandle,
+             pTable->info.tableId, tstrerror(rpcMsg->code), pMsg->rpcMsg.handle);
+      dnodeSendRpcMWriteRsp(pMsg, rpcMsg->code);
     }
   }
 }
@@ -2719,12 +2720,12 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
 
 static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) {
   SAlterTableMsg *pAlter = pMsg->rpcMsg.pCont;
-  mDebug("app:%p:%p, table:%s, alter table msg is received from thandle:%p", pMsg->rpcMsg.ahandle, pMsg,
+  mDebug("msg:%p, app:%p table:%s, alter table msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle,
          pAlter->tableId, pMsg->rpcMsg.handle);
 
   if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pAlter->tableId);
   if (pMsg->pDb == NULL) {
-    mError("app:%p:%p, table:%s, failed to alter table, db not selected", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId);
+    mError("msg:%p, app:%p table:%s, failed to alter table, db not selected", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId);
     return TSDB_CODE_MND_DB_NOT_SELECTED;
   }
   
@@ -2734,13 +2735,13 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) {
   }
 
   if (mnodeCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) {
-    mError("app:%p:%p, table:%s, failed to alter table, its log db", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId);
+    mError("msg:%p, app:%p table:%s, failed to alter table, its log db", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId);
     return TSDB_CODE_MND_MONITOR_DB_FORBIDDEN;
   }
 
   if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pAlter->tableId);
   if (pMsg->pTable == NULL) {
-    mError("app:%p:%p, table:%s, failed to alter table, table not exist", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId);
+    mError("msg:%p, app:%p table:%s, failed to alter table, table not exist", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId);
     return TSDB_CODE_MND_INVALID_TABLE_NAME;
   }
 
@@ -2749,7 +2750,7 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) {
   pAlter->tagValLen = htonl(pAlter->tagValLen);
 
   if (pAlter->numOfCols > 2) {
-    mError("app:%p:%p, table:%s, error numOfCols:%d in alter table", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId,
+    mError("msg:%p, app:%p table:%s, error numOfCols:%d in alter table", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId,
            pAlter->numOfCols);
     return TSDB_CODE_MND_APP_ERROR;
   }
@@ -2760,7 +2761,7 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) {
 
   int32_t code = TSDB_CODE_COM_OPS_NOT_SUPPORT;
   if (pMsg->pTable->type == TSDB_SUPER_TABLE) {
-    mDebug("app:%p:%p, table:%s, start to alter stable", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId);
+    mDebug("msg:%p, app:%p table:%s, start to alter stable", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId);
     if (pAlter->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) {
       code = mnodeAddSuperTableTag(pMsg, pAlter->schema, 1);
     } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) {
@@ -2776,7 +2777,7 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) {
     } else {
     }
   } else {
-    mDebug("app:%p:%p, table:%s, start to alter ctable", pMsg->rpcMsg.ahandle, pMsg, pAlter->tableId);
+    mDebug("msg:%p, app:%p table:%s, start to alter ctable", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId);
     if (pAlter->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) {
       return TSDB_CODE_COM_OPS_NOT_SUPPORT;
     } else if (pAlter->type == TSDB_ALTER_TABLE_ADD_COLUMN) {

src/mnode/src/mnodeUser.c

@@ -42,13 +42,13 @@ static int32_t mnodeProcessAlterUserMsg(SMnodeMsg *pMsg);
 static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg);
 static int32_t mnodeProcessAuthMsg(SMnodeMsg *pMsg);
 
-static int32_t mnodeUserActionDestroy(SSdbOper *pOper) {
-  tfree(pOper->pObj);
+static int32_t mnodeUserActionDestroy(SSdbRow *pRow) {
+  tfree(pRow->pObj);
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeUserActionInsert(SSdbOper *pOper) {
-  SUserObj *pUser = pOper->pObj;
+static int32_t mnodeUserActionInsert(SSdbRow *pRow) {
+  SUserObj *pUser = pRow->pObj;
   SAcctObj *pAcct = mnodeGetAcct(pUser->acct);
 
   if (pAcct != NULL) {
@@ -62,8 +62,8 @@ static int32_t mnodeUserActionInsert(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeUserActionDelete(SSdbOper *pOper) {
-  SUserObj *pUser = pOper->pObj;
+static int32_t mnodeUserActionDelete(SSdbRow *pRow) {
+  SUserObj *pUser = pRow->pObj;
   SAcctObj *pAcct = mnodeGetAcct(pUser->acct);
 
   if (pAcct != NULL) {
@@ -74,8 +74,8 @@ static int32_t mnodeUserActionDelete(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeUserActionUpdate(SSdbOper *pOper) {
-  SUserObj *pUser = pOper->pObj;
+static int32_t mnodeUserActionUpdate(SSdbRow *pRow) {
+  SUserObj *pUser = pRow->pObj;
   SUserObj *pSaved = mnodeGetUser(pUser->user);
   if (pUser != pSaved) {
     memcpy(pSaved, pUser, tsUserUpdateSize);
@@ -85,19 +85,19 @@ static int32_t mnodeUserActionUpdate(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeUserActionEncode(SSdbOper *pOper) {
-  SUserObj *pUser = pOper->pObj;
-  memcpy(pOper->rowData, pUser, tsUserUpdateSize);
-  pOper->rowSize = tsUserUpdateSize;
+static int32_t mnodeUserActionEncode(SSdbRow *pRow) {
+  SUserObj *pUser = pRow->pObj;
+  memcpy(pRow->rowData, pUser, tsUserUpdateSize);
+  pRow->rowSize = tsUserUpdateSize;
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeUserActionDecode(SSdbOper *pOper) {
+static int32_t mnodeUserActionDecode(SSdbRow *pRow) {
   SUserObj *pUser = (SUserObj *)calloc(1, sizeof(SUserObj));
   if (pUser == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
 
-  memcpy(pUser, pOper->rowData, tsUserUpdateSize);
-  pOper->pObj = pUser;
+  memcpy(pUser, pRow->rowData, tsUserUpdateSize);
+  pRow->pObj = pUser;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -123,7 +123,6 @@ static void mnodePrintUserAuth() {
   }
 
   fflush(fp);
-  sdbFreeIter(pIter);
   fclose(fp);
 }
 
@@ -150,25 +149,25 @@ int32_t mnodeInitUsers() {
   SUserObj tObj;
   tsUserUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
 
-  SSdbTableDesc tableDesc = {
-    .tableId      = SDB_TABLE_USER,
-    .tableName    = "users",
+  SSdbTableDesc desc = {
+    .id           = SDB_TABLE_USER,
+    .name         = "users",
     .hashSessions = TSDB_DEFAULT_USERS_HASH_SIZE,
     .maxRowSize   = tsUserUpdateSize,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_STRING,
-    .insertFp     = mnodeUserActionInsert,
-    .deleteFp     = mnodeUserActionDelete,
-    .updateFp     = mnodeUserActionUpdate,
-    .encodeFp     = mnodeUserActionEncode,
-    .decodeFp     = mnodeUserActionDecode,
-    .destroyFp    = mnodeUserActionDestroy,
-    .restoredFp   = mnodeUserActionRestored
+    .fpInsert     = mnodeUserActionInsert,
+    .fpDelete     = mnodeUserActionDelete,
+    .fpUpdate     = mnodeUserActionUpdate,
+    .fpEncode     = mnodeUserActionEncode,
+    .fpDecode     = mnodeUserActionDecode,
+    .fpDestroy    = mnodeUserActionDestroy,
+    .fpRestored   = mnodeUserActionRestored
   };
 
-  tsUserSdb = sdbOpenTable(&tableDesc);
+  tsUserSdb = sdbOpenTable(&desc);
   if (tsUserSdb == NULL) {
-    mError("table:%s, failed to create hash", tableDesc.tableName);
+    mError("table:%s, failed to create hash", desc.name);
     return -1;
   }
 
@@ -177,9 +176,11 @@ int32_t mnodeInitUsers() {
   mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_DROP_USER, mnodeProcessDropUserMsg);
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_USER, mnodeGetUserMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_USER, mnodeRetrieveUsers);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_USER, mnodeCancelGetNextUser);
+
   mnodeAddPeerMsgHandle(TSDB_MSG_TYPE_DM_AUTH, mnodeProcessAuthMsg);
    
-  mDebug("table:%s, hash is created", tableDesc.tableName);
+  mDebug("table:%s, hash is created", desc.name);
   return 0;
 }
 
@@ -196,6 +197,10 @@ void *mnodeGetNextUser(void *pIter, SUserObj **pUser) {
   return sdbFetchRow(tsUserSdb, pIter, (void **)pUser); 
 }
 
+void mnodeCancelGetNextUser(void *pIter) {
+ sdbFreeIter(tsUserSdb, pIter);
+}
+
 void mnodeIncUserRef(SUserObj *pUser) { 
   return sdbIncRef(tsUserSdb, pUser); 
 }
@@ -205,14 +210,14 @@ void mnodeDecUserRef(SUserObj *pUser) {
 }
 
 static int32_t mnodeUpdateUser(SUserObj *pUser, void *pMsg) {
-  SSdbOper oper = {
+  SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,
-    .table = tsUserSdb,
+    .pTable = tsUserSdb,
     .pObj   = pUser,
     .pMsg   = pMsg
   };
 
-  int32_t code = sdbUpdateRow(&oper);
+  int32_t code = sdbUpdateRow(&row);
   if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mError("user:%s, failed to alter by %s, reason:%s", pUser->user, mnodeGetUserFromMsg(pMsg), tstrerror(code));
   } else {
@@ -259,15 +264,15 @@ int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) {
     pUser->superAuth = 1;
   }
 
-  SSdbOper oper = {
+  SSdbRow row = {
     .type     = SDB_OPER_GLOBAL,
-    .table   = tsUserSdb,
+    .pTable   = tsUserSdb,
     .pObj     = pUser,
     .rowSize  = sizeof(SUserObj),
     .pMsg     = pMsg
   };
 
-  code = sdbInsertRow(&oper);
+  code = sdbInsertRow(&row);
   if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mError("user:%s, failed to create by %s, reason:%s", pUser->user, mnodeGetUserFromMsg(pMsg), tstrerror(code));
     tfree(pUser);
@@ -279,14 +284,14 @@ int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) {
 }
 
 static int32_t mnodeDropUser(SUserObj *pUser, void *pMsg) {
-  SSdbOper oper = {
+  SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,
-    .table = tsUserSdb,
+    .pTable = tsUserSdb,
     .pObj   = pUser,
     .pMsg   = pMsg
   };
 
-  int32_t code = sdbDeleteRow(&oper);
+  int32_t code = sdbDeleteRow(&row);
   if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mError("user:%s, failed to drop by %s, reason:%s", pUser->user, mnodeGetUserFromMsg(pMsg), tstrerror(code));
   } else {
@@ -562,20 +567,18 @@ void mnodeDropAllUsers(SAcctObj *pAcct)  {
     if (pUser == NULL) break;
 
     if (strncmp(pUser->acct, pAcct->user, acctNameLen) == 0) {
-      SSdbOper oper = {
+      SSdbRow row = {
         .type   = SDB_OPER_LOCAL,
-        .table = tsUserSdb,
+        .pTable = tsUserSdb,
         .pObj   = pUser,
       };
-      sdbDeleteRow(&oper);
+      sdbDeleteRow(&row);
       numOfUsers++;
     }
 
     mnodeDecUserRef(pUser);
   }
 
-  sdbFreeIter(pIter);
-
   mDebug("acct:%s, all users:%d is dropped from sdb", pAcct->user, numOfUsers);
 }
 

src/mnode/src/mnodeVgroup.c

@@ -72,13 +72,13 @@ static void mnodeDestroyVgroup(SVgObj *pVgroup) {
   tfree(pVgroup);
 }
 
-static int32_t mnodeVgroupActionDestroy(SSdbOper *pOper) {
-  mnodeDestroyVgroup(pOper->pObj);
+static int32_t mnodeVgroupActionDestroy(SSdbRow *pRow) {
+  mnodeDestroyVgroup(pRow->pObj);
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeVgroupActionInsert(SSdbOper *pOper) {
-  SVgObj *pVgroup = pOper->pObj;
+static int32_t mnodeVgroupActionInsert(SSdbRow *pRow) {
+  SVgObj *pVgroup = pRow->pObj;
 
   // refer to db
   SDbObj *pDb = mnodeGetDb(pVgroup->dbName);
@@ -115,8 +115,8 @@ static int32_t mnodeVgroupActionInsert(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeVgroupActionDelete(SSdbOper *pOper) {
-  SVgObj *pVgroup = pOper->pObj;
+static int32_t mnodeVgroupActionDelete(SSdbRow *pRow) {
+  SVgObj *pVgroup = pRow->pObj;
 
   if (pVgroup->pDb == NULL) {
     mError("vgId:%d, db:%s is not exist while insert into hash", pVgroup->vgId, pVgroup->dbName);
@@ -137,8 +137,8 @@ static int32_t mnodeVgroupActionDelete(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeVgroupActionUpdate(SSdbOper *pOper) {
-  SVgObj *pNew = pOper->pObj;
+static int32_t mnodeVgroupActionUpdate(SSdbRow *pRow) {
+  SVgObj *pNew = pRow->pObj;
   SVgObj *pVgroup = mnodeGetVgroup(pNew->vgId);
 
   if (pVgroup != pNew) {
@@ -176,25 +176,25 @@ static int32_t mnodeVgroupActionUpdate(SSdbOper *pOper) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeVgroupActionEncode(SSdbOper *pOper) {
-  SVgObj *pVgroup = pOper->pObj;
-  memcpy(pOper->rowData, pVgroup, tsVgUpdateSize);
-  SVgObj *pTmpVgroup = pOper->rowData;
+static int32_t mnodeVgroupActionEncode(SSdbRow *pRow) {
+  SVgObj *pVgroup = pRow->pObj;
+  memcpy(pRow->rowData, pVgroup, tsVgUpdateSize);
+  SVgObj *pTmpVgroup = pRow->rowData;
   for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) {
     pTmpVgroup->vnodeGid[i].pDnode = NULL;
     pTmpVgroup->vnodeGid[i].role = 0;
   }
 
-  pOper->rowSize = tsVgUpdateSize;
+  pRow->rowSize = tsVgUpdateSize;
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mnodeVgroupActionDecode(SSdbOper *pOper) {
+static int32_t mnodeVgroupActionDecode(SSdbRow *pRow) {
   SVgObj *pVgroup = (SVgObj *) calloc(1, sizeof(SVgObj));
   if (pVgroup == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
 
-  memcpy(pVgroup, pOper->rowData, tsVgUpdateSize);
-  pOper->pObj = pVgroup;
+  memcpy(pVgroup, pRow->rowData, tsVgUpdateSize);
+  pRow->pObj = pVgroup;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -206,23 +206,23 @@ int32_t mnodeInitVgroups() {
   SVgObj tObj;
   tsVgUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
 
-  SSdbTableDesc tableDesc = {
-    .tableId      = SDB_TABLE_VGROUP,
-    .tableName    = "vgroups",
+  SSdbTableDesc desc = {
+    .id           = SDB_TABLE_VGROUP,
+    .name         = "vgroups",
     .hashSessions = TSDB_DEFAULT_VGROUPS_HASH_SIZE,
     .maxRowSize   = tsVgUpdateSize,
     .refCountPos  = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
     .keyType      = SDB_KEY_AUTO,
-    .insertFp     = mnodeVgroupActionInsert,
-    .deleteFp     = mnodeVgroupActionDelete,
-    .updateFp     = mnodeVgroupActionUpdate,
-    .encodeFp     = mnodeVgroupActionEncode,
-    .decodeFp     = mnodeVgroupActionDecode,
-    .destroyFp    = mnodeVgroupActionDestroy,
-    .restoredFp   = mnodeVgroupActionRestored,
+    .fpInsert     = mnodeVgroupActionInsert,
+    .fpDelete     = mnodeVgroupActionDelete,
+    .fpUpdate     = mnodeVgroupActionUpdate,
+    .fpEncode     = mnodeVgroupActionEncode,
+    .fpDecode     = mnodeVgroupActionDecode,
+    .fpDestroy    = mnodeVgroupActionDestroy,
+    .fpRestored   = mnodeVgroupActionRestored,
   };
 
-  tsVgroupSdb = sdbOpenTable(&tableDesc);
+  tsVgroupSdb = sdbOpenTable(&desc);
   if (tsVgroupSdb == NULL) {
     mError("failed to init vgroups data");
     return -1;
@@ -230,6 +230,7 @@ int32_t mnodeInitVgroups() {
 
   mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_VGROUP, mnodeGetVgroupMeta);
   mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VGROUP, mnodeRetrieveVgroups);
+  mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_VGROUP, mnodeCancelGetNextVgroup);
   mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP, mnodeProcessCreateVnodeRsp);
   mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP, mnodeProcessAlterVnodeRsp);
   mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_DROP_VNODE_RSP, mnodeProcessDropVnodeRsp);
@@ -253,13 +254,13 @@ SVgObj *mnodeGetVgroup(int32_t vgId) {
 }
 
 void mnodeUpdateVgroup(SVgObj *pVgroup) {
-  SSdbOper oper = {
+  SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,
-    .table = tsVgroupSdb,
+    .pTable = tsVgroupSdb,
     .pObj   = pVgroup
   };
 
-  int32_t code = sdbUpdateRow(&oper);
+  int32_t code = sdbUpdateRow(&row);
   if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mError("vgId:%d, failed to update vgroup", pVgroup->vgId);
   }
@@ -304,7 +305,7 @@ void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t o
     mnodeDecVgroupRef(pVgroup);
   }
 
-  sdbFreeIter(pIter);
+  mnodeCancelGetNextVgroup(pIter);
 }
 
 void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload) {
@@ -421,7 +422,7 @@ int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSi
 
     int32_t sid = taosAllocateId(pVgroup->idPool);
     if (sid <= 0) {
-      mDebug("app:%p:%p, db:%s, no enough sid in vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pDb->name, pVgroup->vgId);
+      mDebug("msg:%p, app:%p db:%s, no enough sid in vgId:%d", pMsg, pMsg->rpcMsg.ahandle, pDb->name, pVgroup->vgId);
       continue;
     }
 
@@ -442,8 +443,8 @@ int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSi
 
   int32_t code = TSDB_CODE_MND_NO_ENOUGH_DNODES;
   if (pDb->numOfVgroups < maxVgroupsPerDb) {
-    mDebug("app:%p:%p, db:%s, try to create a new vgroup, numOfVgroups:%d maxVgroupsPerDb:%d", pMsg->rpcMsg.ahandle,
-           pMsg, pDb->name, pDb->numOfVgroups, maxVgroupsPerDb);
+    mDebug("msg:%p, app:%p db:%s, try to create a new vgroup, numOfVgroups:%d maxVgroupsPerDb:%d", pMsg,
+           pMsg->rpcMsg.ahandle, pDb->name, pDb->numOfVgroups, maxVgroupsPerDb);
     pthread_mutex_unlock(&pDb->mutex);
     code = mnodeCreateVgroup(pMsg);
     if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) {
@@ -455,8 +456,8 @@ int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSi
 
   if (pDb->numOfVgroups < 1) {
     pthread_mutex_unlock(&pDb->mutex);
-    mDebug("app:%p:%p, db:%s, failed create new vgroup since:%s, numOfVgroups:%d maxVgroupsPerDb:%d ",
-           pMsg->rpcMsg.ahandle, pMsg, pDb->name, tstrerror(code), pDb->numOfVgroups, maxVgroupsPerDb);
+    mDebug("msg:%p, app:%p db:%s, failed create new vgroup since:%s, numOfVgroups:%d maxVgroupsPerDb:%d ", pMsg,
+           pMsg->rpcMsg.ahandle, pDb->name, tstrerror(code), pDb->numOfVgroups, maxVgroupsPerDb);
     return code;
   }
 
@@ -474,7 +475,7 @@ int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSi
 
   int32_t sid = taosAllocateId(pVgroup->idPool);
   if (sid <= 0) {
-    mError("app:%p:%p, db:%s, no enough sid in vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pDb->name, pVgroup->vgId);
+    mError("msg:%p, app:%p db:%s, no enough sid in vgId:%d", pMsg, pMsg->rpcMsg.ahandle, pDb->name, pVgroup->vgId);
     pthread_mutex_unlock(&pDb->mutex);
     return TSDB_CODE_MND_NO_ENOUGH_DNODES;
   }
@@ -491,15 +492,19 @@ void *mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup) {
   return sdbFetchRow(tsVgroupSdb, pIter, (void **)pVgroup); 
 }
 
+void mnodeCancelGetNextVgroup(void *pIter) {
+  sdbFreeIter(tsVgroupSdb, pIter);
+}
+
 static int32_t mnodeCreateVgroupFp(SMnodeMsg *pMsg) {
   SVgObj *pVgroup = pMsg->pVgroup;
   SDbObj *pDb = pMsg->pDb;
   assert(pVgroup);
 
-  mInfo("app:%p:%p, vgId:%d, is created in mnode, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId,
+  mInfo("msg:%p, app:%p vgId:%d, is created in mnode, db:%s replica:%d", pMsg, pMsg->rpcMsg.ahandle, pVgroup->vgId,
         pDb->name, pVgroup->numOfVnodes);
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
-    mInfo("app:%p:%p, vgId:%d, index:%d, dnode:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, i,
+    mInfo("msg:%p, app:%p vgId:%d, index:%d, dnode:%d", pMsg, pMsg->rpcMsg.ahandle, pVgroup->vgId, i,
           pVgroup->vnodeGid[i].dnodeId);
   }
 
@@ -517,30 +522,30 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) {
   assert(pVgroup);
 
   if (code != TSDB_CODE_SUCCESS) {
-    mError("app:%p:%p, vgId:%d, failed to create in sdb, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId,
+    mError("msg:%p, app:%p vgId:%d, failed to create in sdb, reason:%s", pMsg, pMsg->rpcMsg.ahandle, pVgroup->vgId,
            tstrerror(code));
-    SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb};
+    SSdbRow desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .pTable = tsVgroupSdb};
     sdbDeleteRow(&desc);
     return code;
   } else {
-    mInfo("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId,
+    mInfo("msg:%p, app:%p vgId:%d, is created in sdb, db:%s replica:%d", pMsg, pMsg->rpcMsg.ahandle, pVgroup->vgId,
           pDb->name, pVgroup->numOfVnodes);
     pVgroup->status = TAOS_VG_STATUS_READY;
-    SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb};
+    SSdbRow desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .pTable = tsVgroupSdb};
     (void)sdbUpdateRow(&desc);
 
     dnodeReprocessMWriteMsg(pMsg);
     return TSDB_CODE_MND_ACTION_IN_PROGRESS;
     // if (pVgroup->status == TAOS_VG_STATUS_CREATING || pVgroup->status == TAOS_VG_STATUS_READY) {
-    //   mInfo("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId,
+    //   mInfo("msg:%p, app:%p vgId:%d, is created in sdb, db:%s replica:%d", pMsg, pMsg->rpcMsg.ahandle, pVgroup->vgId,
     //         pDb->name, pVgroup->numOfVnodes);
     //   pVgroup->status = TAOS_VG_STATUS_READY;
-    //   SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb};
+    //   SSdbRow desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .pTable = tsVgroupSdb};
     //   (void)sdbUpdateRow(&desc);
     //   dnodeReprocessMWriteMsg(pMsg);
     //   return TSDB_CODE_MND_ACTION_IN_PROGRESS;
     // } else {
-    //   mError("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d, but vgroup is dropping", pMsg->rpcMsg.ahandle,
+    //   mError("msg:%p, app:%p vgId:%d, is created in sdb, db:%s replica:%d, but vgroup is dropping", pMsg->rpcMsg.ahandle,
     //          pMsg, pVgroup->vgId, pDb->name, pVgroup->numOfVnodes);
     //   return TSDB_CODE_MND_VGROUP_NOT_EXIST;
     // }
@@ -571,16 +576,16 @@ int32_t mnodeCreateVgroup(SMnodeMsg *pMsg) {
   pMsg->pVgroup = pVgroup;
   mnodeIncVgroupRef(pVgroup);
 
-  SSdbOper oper = {
+  SSdbRow row = {
     .type     = SDB_OPER_GLOBAL,
-    .table   = tsVgroupSdb,
+    .pTable   = tsVgroupSdb,
     .pObj     = pVgroup,
     .rowSize  = sizeof(SVgObj),
     .pMsg     = pMsg,
-    .reqFp   = mnodeCreateVgroupFp
+    .fpReq    = mnodeCreateVgroupFp
   };
 
-  code = sdbInsertRow(&oper);
+  code = sdbInsertRow(&row);
   if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     pMsg->pVgroup = NULL;
     mnodeDestroyVgroup(pVgroup);
@@ -595,12 +600,12 @@ void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle) {
   } else {
     mDebug("vgId:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes);
     mnodeSendDropVgroupMsg(pVgroup, NULL);
-    SSdbOper oper = {
+    SSdbRow row = {
       .type   = SDB_OPER_GLOBAL,
-      .table = tsVgroupSdb,
+      .pTable = tsVgroupSdb,
       .pObj   = pVgroup
     };
-    sdbDeleteRow(&oper);
+    sdbDeleteRow(&row);
   }
 }
 
@@ -770,7 +775,7 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v
       SDnodeObj * pDnode = pVgroup->vnodeGid[i].pDnode;
       const char *role = "NULL";
       if (pDnode != NULL) {
-        role = mnodeGetMnodeRoleStr(pVgroup->vnodeGid[i].role);
+        role = syncRole[pVgroup->vnodeGid[i].role];
       }
 
       pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
@@ -957,28 +962,28 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
   if (mnodeMsg->received != mnodeMsg->expected) return;
 
   if (mnodeMsg->received == mnodeMsg->successed) {
-     SSdbOper oper = {
+     SSdbRow row = {
       .type    = SDB_OPER_GLOBAL,
-      .table   = tsVgroupSdb,
+      .pTable  = tsVgroupSdb,
       .pObj    = pVgroup,
       .rowSize = sizeof(SVgObj),
       .pMsg    = mnodeMsg,
-      .writeCb = mnodeCreateVgroupCb
+      .fpRsp   = mnodeCreateVgroupCb
     };
 
-    int32_t code = sdbInsertRowImp(&oper);
+    int32_t code = sdbInsertRowToQueue(&row);
     if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
       mnodeMsg->pVgroup = NULL;
       mnodeDestroyVgroup(pVgroup);
       dnodeSendRpcMWriteRsp(mnodeMsg, code);
     }
   } else {
-    SSdbOper oper = {
+    SSdbRow row = {
       .type   = SDB_OPER_GLOBAL,
-      .table = tsVgroupSdb,
+      .pTable = tsVgroupSdb,
       .pObj   = pVgroup
     };
-    sdbDeleteRow(&oper);
+    sdbDeleteRow(&row);
     dnodeSendRpcMWriteRsp(mnodeMsg, mnodeMsg->code);
   }
 }
@@ -1031,12 +1036,12 @@ static void mnodeProcessDropVnodeRsp(SRpcMsg *rpcMsg) {
 
   if (mnodeMsg->received != mnodeMsg->expected) return;
 
-  SSdbOper oper = {
+  SSdbRow row = {
     .type   = SDB_OPER_GLOBAL,
-    .table = tsVgroupSdb,
+    .pTable = tsVgroupSdb,
     .pObj   = pVgroup
   };
-  int32_t code = sdbDeleteRow(&oper);
+  int32_t code = sdbDeleteRow(&row);
   if (code != 0) {
     code = TSDB_CODE_MND_SDB_ERROR;
   }
@@ -1084,19 +1089,17 @@ void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode) {
 
     if (pVgroup->vnodeGid[0].dnodeId == pDropDnode->dnodeId) {
       mnodeDropAllChildTablesInVgroups(pVgroup);
-      SSdbOper oper = {
+      SSdbRow row = {
         .type   = SDB_OPER_LOCAL,
-        .table = tsVgroupSdb,
+        .pTable = tsVgroupSdb,
         .pObj   = pVgroup,
       };
-      sdbDeleteRow(&oper);
+      sdbDeleteRow(&row);
       numOfVgroups++;
     }
     mnodeDecVgroupRef(pVgroup);
   }
 
-  sdbFreeIter(pIter);
-
   mInfo("dnode:%d, all vgroups:%d is dropped from sdb", pDropDnode->dnodeId, numOfVgroups);
 }
 
@@ -1118,8 +1121,6 @@ void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb) {
     mnodeDecVgroupRef(pVgroup);
   }
 
-  sdbFreeIter(pIter);
-
   mInfo("db:%s, all vgroups is updated in sdb", pAlterDb->name);
 }
 #endif
@@ -1135,20 +1136,18 @@ void mnodeDropAllDbVgroups(SDbObj *pDropDb) {
     if (pVgroup == NULL) break;
 
     if (pVgroup->pDb == pDropDb) {
-      SSdbOper oper = {
+      SSdbRow row = {
         .type   = SDB_OPER_LOCAL,
-        .table = tsVgroupSdb,
+        .pTable = tsVgroupSdb,
         .pObj   = pVgroup,
       };
-      sdbDeleteRow(&oper);
+      sdbDeleteRow(&row);
       numOfVgroups++;
     }
 
     mnodeDecVgroupRef(pVgroup);
   }
 
-  sdbFreeIter(pIter);
-
   mInfo("db:%s, all vgroups:%d is dropped from sdb", pDropDb->name, numOfVgroups);
 }
 
@@ -1170,7 +1169,5 @@ void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb) {
     numOfVgroups++;
   }
 
-  sdbFreeIter(pIter);
-
   mInfo("db:%s, all vgroups:%d drop msg is sent to dnode", pDropDb->name, numOfVgroups);
 }

src/mnode/src/mnodeWrite.c

@@ -43,7 +43,7 @@ void mnodeAddWriteMsgHandle(uint8_t msgType, int32_t (*fp)(SMnodeMsg *mnodeMsg))
 
 int32_t mnodeProcessWrite(SMnodeMsg *pMsg) {
   if (pMsg->rpcMsg.pCont == NULL) {
-    mError("app:%p:%p, msg:%s content is null", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType]);
+    mError("msg:%p, app:%p type:%s content is null", pMsg, pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]);
     return TSDB_CODE_MND_INVALID_MSG_LEN;
   }
 
@@ -54,15 +54,15 @@ int32_t mnodeProcessWrite(SMnodeMsg *pMsg) {
     rpcRsp->rsp = epSet;
     rpcRsp->len = sizeof(SRpcEpSet);
 
-    mDebug("app:%p:%p, msg:%s in write queue, will be redirected, numOfEps:%d inUse:%d", pMsg->rpcMsg.ahandle, pMsg,
+    mDebug("msg:%p, app:%p type:%s in write queue, will be redirected, numOfEps:%d inUse:%d", pMsg, pMsg->rpcMsg.ahandle,
            taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse);
     for (int32_t i = 0; i < epSet->numOfEps; ++i) {
       if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) {
         epSet->inUse = (i + 1) % epSet->numOfEps;
-        mDebug("app:%p:%p, mnode index:%d ep:%s:%d, set inUse to %d", pMsg->rpcMsg.ahandle, pMsg, i, epSet->fqdn[i],
+        mDebug("msg:%p, app:%p mnode index:%d ep:%s:%d, set inUse to %d", pMsg, pMsg->rpcMsg.ahandle, i, epSet->fqdn[i],
                htons(epSet->port[i]), epSet->inUse);
       } else {
-        mDebug("app:%p:%p, mnode index:%d ep:%s:%d", pMsg->rpcMsg.ahandle, pMsg, i, epSet->fqdn[i],
+        mDebug("msg:%p, app:%p mnode index:%d ep:%s:%d", pMsg, pMsg->rpcMsg.ahandle, i, epSet->fqdn[i],
                htons(epSet->port[i]));
       }
     }
@@ -71,19 +71,19 @@ int32_t mnodeProcessWrite(SMnodeMsg *pMsg) {
   }
 
   if (tsMnodeProcessWriteMsgFp[pMsg->rpcMsg.msgType] == NULL) {
-    mError("app:%p:%p, msg:%s not processed", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType]);
+    mError("msg:%p, app:%p type:%s not processed", pMsg, pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType]);
     return TSDB_CODE_MND_MSG_NOT_PROCESSED;
   }
 
   int32_t code = mnodeInitMsg(pMsg);
   if (code != TSDB_CODE_SUCCESS) {
-    mError("app:%p:%p, msg:%s not processed, reason:%s", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType],
+    mError("msg:%p, app:%p type:%s not processed, reason:%s", pMsg, pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType],
            tstrerror(code));
     return code;
   }
 
   if (!pMsg->pUser->writeAuth) {
-    mError("app:%p:%p, msg:%s not processed, no write auth", pMsg->rpcMsg.ahandle, pMsg,
+    mError("msg:%p, app:%p type:%s not processed, no write auth", pMsg, pMsg->rpcMsg.ahandle,
            taosMsg[pMsg->rpcMsg.msgType]);
     return TSDB_CODE_MND_NO_RIGHTS;
   }

src/os/inc/osAlpine.h

@@ -77,6 +77,7 @@ extern "C" {
 #include <sys/utsname.h>
 #include <sys/resource.h>
 #include <linux/sysctl.h>
+#include <math.h>
 
 typedef int(*__compar_fn_t)(const void *, const void *);
 void  error (int, int, const char *);

src/os/inc/osArm32.h

@@ -76,6 +76,7 @@ extern "C" {
 #include <sys/utsname.h>
 #include <sys/resource.h>
 #include <error.h>
+#include <math.h>
 
 #define TAOS_OS_FUNC_LZ4
   #define BUILDIN_CLZL(val) __builtin_clzll(val)

src/os/inc/osArm64.h

@@ -77,6 +77,7 @@ extern "C" {
 #include <sys/resource.h>
 #include <error.h>
 #include <linux/sysctl.h>
+#include <math.h>
 
 #ifdef __cplusplus
 }

src/os/inc/osDarwin.h

@@ -70,6 +70,7 @@ extern "C" {
 #include <dispatch/dispatch.h>
 #include <fcntl.h>
 #include <sys/utsname.h>
+#include <math.h>
 
 #define TAOS_OS_FUNC_FILE_SENDIFLE
 

src/os/inc/osFile.h

@@ -20,17 +20,6 @@
 extern "C" {
 #endif
 
-#define tread(fd, buf, count) read(fd, buf, count)
-#define twrite(fd, buf, count) write(fd, buf, count)
-#define tlseek(fd, offset, whence) lseek(fd, offset, whence)
-#define tclose(fd)    \
-  {                       \
-    if (FD_VALID(fd)) {    \
-      close(fd);           \
-      fd = FD_INITIALIZER; \
-    }                     \
-  }
-
 int64_t taosReadImp(int32_t fd, void *buf, int64_t count);
 int64_t taosWriteImp(int32_t fd, void *buf, int64_t count);
 int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence);
@@ -39,7 +28,13 @@ int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstP
 #define taosRead(fd, buf, count) taosReadImp(fd, buf, count)
 #define taosWrite(fd, buf, count) taosWriteImp(fd, buf, count)
 #define taosLSeek(fd, offset, whence) taosLSeekImp(fd, offset, whence)
-#define taosClose(x) tclose(x)
+#define taosClose(fd)      \
+  {                        \
+    if (FD_VALID(fd)) {    \
+      close(fd);           \
+      fd = FD_INITIALIZER; \
+    }                      \
+  }
 
 // TAOS_OS_FUNC_FILE_SENDIFLE
 int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size);

src/os/inc/osLinux32.h

@@ -76,6 +76,7 @@ extern "C" {
 #include <sys/utsname.h>
 #include <sys/resource.h>
 #include <error.h>
+#include <math.h>
 
 #define TAOS_OS_FUNC_LZ4
 #define BUILDIN_CLZL(val) __builtin_clzll(val)

src/os/inc/osLinux64.h

@@ -79,6 +79,7 @@ extern "C" {
 #include <error.h>
 #endif
 #include <linux/sysctl.h>
+#include <math.h>
 
 #ifdef __cplusplus
 }

src/os/inc/osWindows.h

@@ -40,6 +40,7 @@
 #include <time.h>
 #include <inttypes.h>
 #include <conio.h>
+#include <math.h>
 #include "msvcProcess.h"
 #include "msvcDirect.h"
 #include "msvcFcntl.h"

src/os/src/detail/osFile.c

@@ -116,7 +116,7 @@ int64_t taosWriteImp(int32_t fd, void *buf, int64_t n) {
 }
 
 int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) {
-  return (int64_t)tlseek(fd, (long)offset, whence);
+  return (int64_t)lseek(fd, (long)offset, whence);
 }
 
 #ifndef TAOS_OS_FUNC_FILE_SENDIFLE

src/plugins/http/inc/httpInt.h

@@ -118,7 +118,7 @@ typedef struct {
 
 typedef struct {
   char *module;
-  bool (*decodeFp)(struct HttpContext *pContext);
+  bool (*fpDecode)(struct HttpContext *pContext);
 } HttpDecodeMethod;
 
 typedef struct {

src/plugins/http/src/httpHandle.c

@@ -21,11 +21,11 @@
 #include "httpHandle.h"
 
 bool httpDecodeRequest(HttpContext* pContext) {
-  if (pContext->decodeMethod->decodeFp == NULL) {
+  if (pContext->decodeMethod->fpDecode == NULL) {
     return false;
   }
 
-  return (*pContext->decodeMethod->decodeFp)(pContext);
+  return (*pContext->decodeMethod->fpDecode)(pContext);
 }
 
 /**

src/query/inc/qUtil.h

@@ -27,7 +27,7 @@
 int32_t getOutputInterResultBufSize(SQuery* pQuery);
 
 void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow, int16_t type);
-void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src);
+void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src, int16_t type);
 SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index);
 
 int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, int32_t size, int32_t threshold, int16_t type);

src/query/src/qExecutor.c

@@ -6076,9 +6076,9 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
   pQueryMsg->interval.interval = htobe64(pQueryMsg->interval.interval);
   pQueryMsg->interval.sliding = htobe64(pQueryMsg->interval.sliding);
   pQueryMsg->interval.offset = htobe64(pQueryMsg->interval.offset);
-  pQueryMsg->interval.intervalUnit = pQueryMsg->interval.intervalUnit;
-  pQueryMsg->interval.slidingUnit = pQueryMsg->interval.slidingUnit;
-  pQueryMsg->interval.offsetUnit = pQueryMsg->interval.offsetUnit;
+  // pQueryMsg->interval.intervalUnit = pQueryMsg->interval.intervalUnit;
+  // pQueryMsg->interval.slidingUnit = pQueryMsg->interval.slidingUnit;
+  // pQueryMsg->interval.offsetUnit = pQueryMsg->interval.offsetUnit;
   pQueryMsg->limit = htobe64(pQueryMsg->limit);
   pQueryMsg->offset = htobe64(pQueryMsg->offset);
 

src/query/src/qResultbuf.c

@@ -423,9 +423,8 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
   unlink(pResultBuf->path);
   tfree(pResultBuf->path);
 
-  SHashMutableIterator* iter = taosHashCreateIter(pResultBuf->groupSet);
-  while(taosHashIterNext(iter)) {
-    SArray** p = (SArray**) taosHashIterGet(iter);
+  SArray** p = taosHashIterate(pResultBuf->groupSet, NULL);
+  while(p) {
     size_t n = taosArrayGetSize(*p);
     for(int32_t i = 0; i < n; ++i) {
       SPageInfo* pi = taosArrayGetP(*p, i);
@@ -434,10 +433,9 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
     }
 
     taosArrayDestroy(*p);
+    p = taosHashIterate(pResultBuf->groupSet, p);
   }
 
-  taosHashDestroyIter(iter);
-
   tdListFree(pResultBuf->lruList);
   taosArrayDestroy(pResultBuf->emptyDummyIdList);
   taosHashCleanup(pResultBuf->groupSet);

src/query/src/qUtil.c

@@ -133,7 +133,7 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
   
   // clear all the closed windows from the window list
   for (int32_t k = 0; k < remain; ++k) {
-    copyResultRow(pRuntimeEnv, pWindowResInfo->pResult[k], pWindowResInfo->pResult[num + k]);
+    copyResultRow(pRuntimeEnv, pWindowResInfo->pResult[k], pWindowResInfo->pResult[num + k], type);
   }
   
   // move the unclosed window in the front of the window list
@@ -272,9 +272,15 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes, int16
  * since the attribute of "Pos" is bound to each window result when the window result is created in the
  * disk-based result buffer.
  */
-void copyResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *dst, const SResultRow *src) {
+void copyResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *dst, const SResultRow *src, int16_t type) {
   dst->numOfRows = src->numOfRows;
+
+  if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
+    dst->key = realloc(dst->key, varDataTLen(src->key));
+    varDataCopy(dst->key, src->key);
+  } else {
     dst->win   = src->win;
+  }
   dst->closed = src->closed;
   
   int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput;

src/rpc/src/rpcTcp.c

@@ -371,10 +371,13 @@ void taosCloseTcpConnection(void *chandle) {
 
 int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) {
   SFdObj *pFdObj = chandle;
-
   if (pFdObj == NULL || pFdObj->signature != pFdObj) return -1;
+  SThreadObj *pThreadObj = pFdObj->pThreadObj;
+
+  int ret = taosWriteMsg(pFdObj->fd, data, len);
+  tTrace("%s %p TCP data is sent, FD:%p fd:%d bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, ret); 
 
-  return taosWriteMsg(pFdObj->fd, data, len);
+  return ret;
 }
 
 static void taosReportBrokenLink(SFdObj *pFdObj) {
@@ -409,7 +412,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
 
   headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead));
   if (headLen != sizeof(SRpcHead)) {
-    tDebug("%s %p read error, headLen:%d", pThreadObj->label, pFdObj->thandle, headLen);
+    tDebug("%s %p read error, FD:%p headLen:%d", pThreadObj->label, pFdObj->thandle, pFdObj, headLen);
     return -1; 
   }
 
@@ -420,7 +423,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
     tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen);
     return -1;
   } else {
-    tTrace("TCP malloc mem:%p size:%d", buffer, size);
+    tTrace("%s %p read data, FD:%p fd:%d TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, buffer);
   }
 
   msg = buffer + tsRpcOverhead;
@@ -583,8 +586,8 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
 
   pthread_mutex_unlock(&pThreadObj->mutex);
 
-  tDebug("%s %p TCP connection is closed, FD:%p numOfFds:%d", 
-          pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds);
+  tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d", 
+          pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, pThreadObj->numOfFds);
 
   tfree(pFdObj);
 }

src/sync/inc/syncInt.h

@@ -27,16 +27,22 @@ extern "C" {
 #define sDebug(...) { if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }}
 #define sTrace(...) { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }}
 
-#define TAOS_SMSG_SYNC_DATA    1
-#define TAOS_SMSG_FORWARD      2
-#define TAOS_SMSG_FORWARD_RSP  3
-#define TAOS_SMSG_SYNC_REQ     4 
-#define TAOS_SMSG_SYNC_RSP     5
-#define TAOS_SMSG_SYNC_MUST    6
-#define TAOS_SMSG_STATUS       7
+typedef enum {
+  TAOS_SMSG_SYNC_DATA     = 1,
+  TAOS_SMSG_FORWARD       = 2,
+  TAOS_SMSG_FORWARD_RSP   = 3,
+  TAOS_SMSG_SYNC_REQ      = 4,
+  TAOS_SMSG_SYNC_RSP      = 5,
+  TAOS_SMSG_SYNC_MUST     = 6,
+  TAOS_SMSG_STATUS        = 7,
+  TAOS_SMSG_SYNC_DATA_RSP = 8,
+} ESyncMsgType;
 
 #define SYNC_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + sizeof(SSyncHead) + 16)
 #define SYNC_RECV_BUFFER_SIZE (5*1024*1024)
+#define SYNC_FWD_TIMER  300
+#define SYNC_ROLE_TIMER 10000
+#define SYNC_WAIT_AFTER_CHOOSE_MASTER 3
 
 #define nodeRole    pNode->peerInfo[pNode->selfIndex]->role
 #define nodeVersion pNode->peerInfo[pNode->selfIndex]->version
@@ -60,6 +66,10 @@ typedef struct {
   int32_t   sourceId;  // only for arbitrator
 } SFirstPkt;
 
+typedef struct {
+  int8_t sync;
+} SFirstPktRsp;
+
 typedef struct {
   int8_t    role;
   uint64_t  version;
@@ -68,6 +78,9 @@ typedef struct {
 typedef struct {
   int8_t      role;
   int8_t      ack;
+  int8_t      type;
+  int8_t      reserved[3];
+  uint16_t    tranId;
   uint64_t    version;
   SPeerStatus peersStatus[];
 } SPeersStatus;
@@ -120,12 +133,13 @@ typedef struct SsyncPeer {
   int32_t  nodeId;
   uint32_t ip;
   uint16_t port;
-  char     fqdn[TSDB_FQDN_LEN];   // peer ip string
-  char     id[TSDB_EP_LEN + 32];  // peer vgId + end point
   int8_t   role;
   int8_t   sstatus;               // sync status
+  char     fqdn[TSDB_FQDN_LEN];   // peer ip string
+  char     id[TSDB_EP_LEN + 32];  // peer vgId + end point
   uint64_t version;
   uint64_t sversion;        // track the peer version in retrieve process
+  uint64_t lastVer;         // track the file version while retrieve
   int32_t  syncFd;
   int32_t  peerFd;          // forward FD
   int32_t  numOfRetrieves;  // number of retrieves tried
@@ -135,7 +149,7 @@ typedef struct SsyncPeer {
   int32_t  notifyFd;
   int32_t  watchNum;
   int32_t *watchFd;
-  int8_t   refCount;  // reference count
+  int32_t  refCount;  // reference count
   struct   SSyncNode *pSyncNode;
 } SSyncPeer;
 
@@ -143,16 +157,15 @@ typedef struct SSyncNode {
   char         path[TSDB_FILENAME_LEN];
   int8_t       replica;
   int8_t       quorum;
+  int8_t       selfIndex;
   uint32_t     vgId;
   int64_t      rid;
-  void        *ahandle;
-  int8_t       selfIndex;
-  SSyncPeer   *peerInfo[TAOS_SYNC_MAX_REPLICA+1];  // extra one for arbitrator
-  SSyncPeer   *pMaster;
-  int8_t       refCount;
+  SSyncPeer *  peerInfo[TAOS_SYNC_MAX_REPLICA + 1];  // extra one for arbitrator
+  SSyncPeer *  pMaster;
   SRecvBuffer *pRecv;
-  SSyncFwds   *pSyncFwds;  // saved forward info if quorum >1
-  void        *pFwdTimer;
+  SSyncFwds *  pSyncFwds;  // saved forward info if quorum >1
+  void *       pFwdTimer;
+  void *       pRoleTimer;
   FGetFileInfo      getFileInfo;
   FGetWalInfo       getWalInfo;
   FWriteToCache     writeToCache;
@@ -160,12 +173,14 @@ typedef struct SSyncNode {
   FNotifyRole       notifyRole;
   FNotifyFlowCtrl   notifyFlowCtrl;
   FNotifyFileSynced notifyFileSynced;
+  FGetFileVersion   getFileVersion;
   pthread_mutex_t   mutex;
 } SSyncNode;
 
 // sync module global
 extern int32_t tsSyncNum;
 extern char    tsNodeFqdn[TSDB_FQDN_LEN];
+extern char *  syncStatus[];
 
 void *syncRetrieveData(void *param);
 void *syncRestoreData(void *param);

src/sync/src/syncMain.c

@@ -13,8 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-//#include <stdint.h>
-//#include <stdbool.h>
+#define _DEFAULT_SOURCE
 #include "os.h"
 #include "hash.h"
 #include "tlog.h"
@@ -50,7 +49,7 @@ static int32_t  tsSyncRefId = -1;
 static void    syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer);
 static void    syncRecoverFromMaster(SSyncPeer *pPeer);
 static void    syncCheckPeerConnection(void *param, void *tmrId);
-static void    syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack);
+static void    syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId);
 static void    syncProcessBrokenLink(void *param);
 static int32_t syncProcessPeerMsg(void *param, void *buffer);
 static void    syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp);
@@ -59,6 +58,7 @@ static void    syncAddArbitrator(SSyncNode *pNode);
 static void    syncFreeNode(void *);
 static void    syncRemoveConfirmedFwdInfo(SSyncNode *pNode);
 static void    syncMonitorFwdInfos(void *param, void *tmrId);
+static void    syncMonitorNodeRole(void *param, void *tmrId);
 static void    syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code);
 static void    syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle);
 static void    syncRestartPeer(SSyncPeer *pPeer);
@@ -73,8 +73,42 @@ char* syncRole[] = {
   "master"
 };
 
+char *syncStatus[] = {
+  "init",
+  "start",
+  "file",
+  "cache",
+  "invalid"
+};
+
+typedef enum {
+  SYNC_STATUS_BROADCAST,
+  SYNC_STATUS_BROADCAST_RSP,
+  SYNC_STATUS_SETUP_CONN,
+  SYNC_STATUS_SETUP_CONN_RSP,
+  SYNC_STATUS_EXCHANGE_DATA,
+  SYNC_STATUS_EXCHANGE_DATA_RSP,
+  SYNC_STATUS_CHECK_ROLE,
+  SYNC_STATUS_CHECK_ROLE_RSP
+} ESyncStatusType;
+
+char *statusType[] = {
+  "broadcast",
+  "broadcast-rsp",
+  "setup-conn",
+  "setup-conn-rsp",
+  "exchange-data",
+  "exchange-data-rsp",
+  "check-role",
+  "check-role-rsp"
+};
+
+uint16_t syncGenTranId() {
+  return taosRand() & 0XFFFF;
+}
+
 int32_t syncInit() {
-  SPoolInfo info;
+  SPoolInfo info = {0};
 
   info.numOfThreads = tsSyncTcpThreads;
   info.serverIp = 0;
@@ -98,7 +132,7 @@ int32_t syncInit() {
     return -1;
   }
 
-  tsVgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true);
+  tsVgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
   if (tsVgIdHash == NULL) {
     sError("failed to init tsVgIdHash");
     taosTmrCleanUp(tsSyncTmrCtrl);
@@ -145,7 +179,7 @@ void syncCleanUp() {
 int64_t syncStart(const SSyncInfo *pInfo) {
   const SSyncCfg *pCfg = &pInfo->syncCfg;
 
-  SSyncNode *pNode = (SSyncNode *)calloc(sizeof(SSyncNode), 1);
+  SSyncNode *pNode = calloc(sizeof(SSyncNode), 1);
   if (pNode == NULL) {
     sError("no memory to allocate syncNode");
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -155,7 +189,6 @@ int64_t syncStart(const SSyncInfo *pInfo) {
   tstrncpy(pNode->path, pInfo->path, sizeof(pNode->path));
   pthread_mutex_init(&pNode->mutex, NULL);
 
-  pNode->ahandle = pInfo->ahandle;
   pNode->getFileInfo = pInfo->getFileInfo;
   pNode->getWalInfo = pInfo->getWalInfo;
   pNode->writeToCache = pInfo->writeToCache;
@@ -163,6 +196,7 @@ int64_t syncStart(const SSyncInfo *pInfo) {
   pNode->confirmForward = pInfo->confirmForward;
   pNode->notifyFlowCtrl = pInfo->notifyFlowCtrl;
   pNode->notifyFileSynced = pInfo->notifyFileSynced;
+  pNode->getFileVersion = pInfo->getFileVersion;
 
   pNode->selfIndex = -1;
   pNode->vgId = pInfo->vgId;
@@ -176,18 +210,18 @@ int64_t syncStart(const SSyncInfo *pInfo) {
     return -1;
   }
 
-  for (int32_t i = 0; i < pCfg->replica; ++i) {
-    const SNodeInfo *pNodeInfo = pCfg->nodeInfo + i;
-    pNode->peerInfo[i] = syncAddPeer(pNode, pNodeInfo);
-    if (pNode->peerInfo[i] == NULL) {
-      sError("vgId:%d, node:%d fqdn:%s port:%u is not configured, stop taosd", pNode->vgId, pNodeInfo->nodeId, pNodeInfo->nodeFqdn,
-             pNodeInfo->nodePort);
+  for (int32_t index = 0; index < pCfg->replica; ++index) {
+    const SNodeInfo *pNodeInfo = pCfg->nodeInfo + index;
+    pNode->peerInfo[index] = syncAddPeer(pNode, pNodeInfo);
+    if (pNode->peerInfo[index] == NULL) {
+      sError("vgId:%d, node:%d fqdn:%s port:%u is not configured, stop taosd", pNode->vgId, pNodeInfo->nodeId,
+             pNodeInfo->nodeFqdn, pNodeInfo->nodePort);
       syncStop(pNode->rid);
       exit(1);
     }
 
     if ((strcmp(pNodeInfo->nodeFqdn, tsNodeFqdn) == 0) && (pNodeInfo->nodePort == tsSyncPort)) {
-      pNode->selfIndex = i;
+      pNode->selfIndex = index;
     }
   }
 
@@ -211,18 +245,25 @@ int64_t syncStart(const SSyncInfo *pInfo) {
     return -1;
   }
 
-  pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, tsSyncTmrCtrl);
+  pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, SYNC_FWD_TIMER, (void *)pNode->rid, tsSyncTmrCtrl);
   if (pNode->pFwdTimer == NULL) {
-    sError("vgId:%d, failed to allocate timer", pNode->vgId);
+    sError("vgId:%d, failed to allocate fwd timer", pNode->vgId);
+    syncStop(pNode->rid);
+    return -1;
+  }
+
+  pNode->pRoleTimer = taosTmrStart(syncMonitorNodeRole, SYNC_ROLE_TIMER, (void *)pNode->rid, tsSyncTmrCtrl);
+  if (pNode->pRoleTimer == NULL) {
+    sError("vgId:%d, failed to allocate role timer", pNode->vgId);
     syncStop(pNode->rid);
     return -1;
   }
 
   syncAddArbitrator(pNode);
-  taosHashPut(tsVgIdHash, (const char *)&pNode->vgId, sizeof(int32_t), (char *)(&pNode), sizeof(SSyncNode *));
+  taosHashPut(tsVgIdHash, &pNode->vgId, sizeof(int32_t), &pNode, sizeof(SSyncNode *));
 
   if (pNode->notifyRole) {
-    (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+    (*pNode->notifyRole)(pNode->vgId, nodeRole);
   }
 
   return pNode->rid;
@@ -236,20 +277,21 @@ void syncStop(int64_t rid) {
 
   sInfo("vgId:%d, cleanup sync", pNode->vgId);
 
-  pthread_mutex_lock(&(pNode->mutex));
+  pthread_mutex_lock(&pNode->mutex);
 
-  if (tsVgIdHash) taosHashRemove(tsVgIdHash, (const char *)&pNode->vgId, sizeof(int32_t));
+  if (tsVgIdHash) taosHashRemove(tsVgIdHash, &pNode->vgId, sizeof(int32_t));
   if (pNode->pFwdTimer) taosTmrStop(pNode->pFwdTimer);
+  if (pNode->pRoleTimer) taosTmrStop(pNode->pRoleTimer);
 
-  for (int32_t i = 0; i < pNode->replica; ++i) {
-    pPeer = pNode->peerInfo[i];
+  for (int32_t index = 0; index < pNode->replica; ++index) {
+    pPeer = pNode->peerInfo[index];
     if (pPeer) syncRemovePeer(pPeer);
   }
 
   pPeer = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA];
   if (pPeer) syncRemovePeer(pPeer);
 
-  pthread_mutex_unlock(&(pNode->mutex));
+  pthread_mutex_unlock(&pNode->mutex);
 
   taosReleaseRef(tsSyncRefId, rid);
   taosRemoveRef(tsSyncRefId, rid);
@@ -264,7 +306,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) {
   sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], pNewCfg->replica,
         pNode->replica);
 
-  pthread_mutex_lock(&(pNode->mutex));
+  pthread_mutex_lock(&pNode->mutex);
 
   for (i = 0; i < pNode->replica; ++i) {
     for (j = 0; j < pNewCfg->replica; ++j) {
@@ -314,17 +356,15 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) {
   if (pNewCfg->replica <= 1) {
     sInfo("vgId:%d, no peers are configured, work as master!", pNode->vgId);
     nodeRole = TAOS_SYNC_ROLE_MASTER;
-    (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+    (*pNode->notifyRole)(pNode->vgId, nodeRole);
   }
 
-  pthread_mutex_unlock(&(pNode->mutex));
+  pthread_mutex_unlock(&pNode->mutex);
 
-  sInfo("vgId:%d, %d replicas are configured, quorum:%d role:%s", pNode->vgId, pNode->replica, pNode->quorum,
-        syncRole[nodeRole]);
+  sInfo("vgId:%d, %d replicas are configured, quorum:%d", pNode->vgId, pNode->replica, pNode->quorum);
   syncBroadcastStatus(pNode);
 
   taosReleaseRef(tsSyncRefId, rid);
-
   return 0;
 }
 
@@ -356,10 +396,10 @@ void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {
     pFwdRsp->code = code;
 
     int32_t msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp);
-    int32_t retLen = write(pPeer->peerFd, msg, msgLen);
+    int32_t retLen = taosWriteMsg(pPeer->peerFd, msg, msgLen);
 
     if (retLen == msgLen) {
-      sDebug("%s, forward-rsp is sent, code:%x hver:%" PRIu64, pPeer->id, code, version);
+      sTrace("%s, forward-rsp is sent, code:%x hver:%" PRIu64, pPeer->id, code, version);
     } else {
       sDebug("%s, failed to send forward ack, restart", pPeer->id);
       syncRestartConnection(pPeer);
@@ -369,6 +409,7 @@ void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {
   taosReleaseRef(tsSyncRefId, rid);
 }
 
+#if 0
 void syncRecover(int64_t rid) {
   SSyncPeer *pPeer;
 
@@ -379,22 +420,23 @@ void syncRecover(int64_t rid) {
   // if take this node to unsync state, the whole system may not work
 
   nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
-  (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+  (*pNode->notifyRole)(pNode->vgId, nodeRole);
   nodeVersion = 0;
 
-  pthread_mutex_lock(&(pNode->mutex));
+  pthread_mutex_lock(&pNode->mutex);
 
   for (int32_t i = 0; i < pNode->replica; ++i) {
-    pPeer = (SSyncPeer *)pNode->peerInfo[i];
+    pPeer = pNode->peerInfo[i];
     if (pPeer->peerFd >= 0) {
       syncRestartConnection(pPeer);
     }
   }
 
-  pthread_mutex_unlock(&(pNode->mutex));
+  pthread_mutex_unlock(&pNode->mutex);
 
   taosReleaseRef(tsSyncRefId, rid);
 }
+#endif
 
 int32_t syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) {
   SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
@@ -449,10 +491,10 @@ static void syncFreeNode(void *param) {
   tfree(pNode);
 }
 
-void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_8(&pPeer->refCount, 1); }
+void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_32(&pPeer->refCount, 1); }
 
 int32_t syncDecPeerRef(SSyncPeer *pPeer) {
-  if (atomic_sub_fetch_8(&pPeer->refCount, 1) == 0) {
+  if (atomic_sub_fetch_32(&pPeer->refCount, 1) == 0) {
     taosReleaseRef(tsSyncRefId, pPeer->pSyncNode->rid);
 
     sDebug("%s, resource is freed", pPeer->id);
@@ -465,6 +507,8 @@ int32_t syncDecPeerRef(SSyncPeer *pPeer) {
 }
 
 static void syncClosePeerConn(SSyncPeer *pPeer) {
+  sDebug("%s, pfd:%d sfd:%d will be closed", pPeer->id, pPeer->peerFd, pPeer->syncFd);
+
   taosTmrStopA(&pPeer->timer);
   taosClose(pPeer->syncFd);
   if (pPeer->peerFd >= 0) {
@@ -497,7 +541,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
   pPeer->ip = ip;
   pPeer->port = pInfo->nodePort;
   pPeer->fqdn[sizeof(pPeer->fqdn) - 1] = 0;
-  snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d, peer:%s:%u", pNode->vgId, pPeer->fqdn, pPeer->port);
+  snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d, nodeId:%d", pNode->vgId, pPeer->nodeId);
 
   pPeer->peerFd = -1;
   pPeer->syncFd = -1;
@@ -510,7 +554,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
   if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) {
     int32_t checkMs = 100 + (pNode->vgId * 10) % 100;
     if (pNode->vgId > 1) checkMs = tsStatusInterval * 1000 + checkMs;
-    sDebug("%s, start to check peer connection after %d ms", pPeer->id, checkMs);
+    sDebug("%s, check peer connection after %d ms", pPeer->id, checkMs);
     taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, tsSyncTmrCtrl, &pPeer->timer);
   }
 
@@ -519,22 +563,20 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
 }
 
 void syncBroadcastStatus(SSyncNode *pNode) {
-  SSyncPeer *pPeer;
-
-  for (int32_t i = 0; i < pNode->replica; ++i) {
-    if (i == pNode->selfIndex) continue;
-    pPeer = pNode->peerInfo[i];
-    syncSendPeersStatusMsgToPeer(pPeer, 1);
+  for (int32_t index = 0; index < pNode->replica; ++index) {
+    if (index == pNode->selfIndex) continue;
+    SSyncPeer *pPeer = pNode->peerInfo[index];
+    syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_BROADCAST, syncGenTranId());
   }
 }
 
 static void syncResetFlowCtrl(SSyncNode *pNode) {
-  for (int32_t i = 0; i < pNode->replica; ++i) {
-    pNode->peerInfo[i]->numOfRetrieves = 0;
+  for (int32_t index = 0; index < pNode->replica; ++index) {
+    pNode->peerInfo[index]->numOfRetrieves = 0;
   }
 
   if (pNode->notifyFlowCtrl) {
-    (*pNode->notifyFlowCtrl)(pNode->ahandle, 0);
+    (*pNode->notifyFlowCtrl)(pNode->vgId, 0);
   }
 }
 
@@ -544,8 +586,6 @@ static void syncChooseMaster(SSyncNode *pNode) {
   int32_t    index = -1;
   int32_t    replica = pNode->replica;
 
-  sDebug("vgId:%d, choose master", pNode->vgId);
-
   for (int32_t i = 0; i < pNode->replica; ++i) {
     if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE) {
       onlineNum++;
@@ -587,19 +627,11 @@ static void syncChooseMaster(SSyncNode *pNode) {
       sInfo("vgId:%d, start to work as master", pNode->vgId);
       nodeRole = TAOS_SYNC_ROLE_MASTER;
 
-#if 0
-      for (int32_t i = 0; i < pNode->replica; ++i) {
-        if (i == index) continue;
-        pPeer = pNode->peerInfo[i];
-        if (pPeer->version == nodeVersion) {
-          pPeer->role = TAOS_SYNC_ROLE_SLAVE;
-          pPeer->sstatus = TAOS_SYNC_STATUS_CACHE;
-          sInfo("%s, it shall work as slave", pPeer->id);
-        }
-      }
-#endif
+      // Wait for other nodes to receive status to avoid version inconsistency
+      taosMsleep(SYNC_WAIT_AFTER_CHOOSE_MASTER);
+
       syncResetFlowCtrl(pNode);
-      (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+      (*pNode->notifyRole)(pNode->vgId, nodeRole);
     } else {
       pPeer = pNode->peerInfo[index];
       sInfo("%s, it shall work as master", pPeer->id);
@@ -611,11 +643,11 @@ static void syncChooseMaster(SSyncNode *pNode) {
 
 static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
   int32_t onlineNum = 0;
-  int32_t index = -1;
+  int32_t masterIndex = -1;
   int32_t replica = pNode->replica;
 
-  for (int32_t i = 0; i < pNode->replica; ++i) {
-    if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE) {
+  for (int32_t index = 0; index < pNode->replica; ++index) {
+    if (pNode->peerInfo[index]->role != TAOS_SYNC_ROLE_OFFLINE) {
       onlineNum++;
     }
   }
@@ -630,27 +662,26 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
   if (onlineNum <= replica * 0.5) {
     if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) {
       nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
-      // pNode->peerInfo[pNode->selfIndex]->role = nodeRole;
-      (*pNode->notifyRole)(pNode->ahandle, nodeRole);
-      sInfo("vgId:%d, change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica);
+      (*pNode->notifyRole)(pNode->vgId, nodeRole);
+      sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica);
     }
   } else {
-    for (int32_t i = 0; i < pNode->replica; ++i) {
-      SSyncPeer *pTemp = pNode->peerInfo[i];
+    for (int32_t index = 0; index < pNode->replica; ++index) {
+      SSyncPeer *pTemp = pNode->peerInfo[index];
       if (pTemp->role != TAOS_SYNC_ROLE_MASTER) continue;
-      if (index < 0) {
-        index = i;
+      if (masterIndex < 0) {
+        masterIndex = index;
       } else {  // multiple masters, it shall not happen
-        if (i == pNode->selfIndex) {
+        if (masterIndex == pNode->selfIndex) {
           sError("%s, peer is master, work as slave instead", pTemp->id);
           nodeRole = TAOS_SYNC_ROLE_SLAVE;
-          (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+          (*pNode->notifyRole)(pNode->vgId, nodeRole);
         }
       }
     }
   }
 
-  SSyncPeer *pMaster = (index >= 0) ? pNode->peerInfo[index] : NULL;
+  SSyncPeer *pMaster = (masterIndex >= 0) ? pNode->peerInfo[masterIndex] : NULL;
   return pMaster;
 }
 
@@ -659,68 +690,71 @@ static int32_t syncValidateMaster(SSyncPeer *pPeer) {
   int32_t code = 0;
 
   if (nodeRole == TAOS_SYNC_ROLE_MASTER && nodeVersion < pPeer->version) {
-    sDebug("%s, slave has higher version, restart all connections!!!", pPeer->id);
+    sDebug("%s, peer has higher sver:%" PRIu64 ", restart all peer connections", pPeer->id, pPeer->version);
     nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
-    (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+    (*pNode->notifyRole)(pNode->vgId, nodeRole);
     code = -1;
 
-    for (int32_t i = 0; i < pNode->replica; ++i) {
-      if (i == pNode->selfIndex) continue;
-      syncRestartPeer(pNode->peerInfo[i]);
+    for (int32_t index = 0; index < pNode->replica; ++index) {
+      if (index == pNode->selfIndex) continue;
+      syncRestartPeer(pNode->peerInfo[index]);
     }
   }
 
   return code;
 }
 
-static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t newRole) {
+static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t newPeerRole) {
   SSyncNode *pNode = pPeer->pSyncNode;
-  int8_t     peerOldRole = pPeer->role;
-  int8_t     selfOldRole = nodeRole;
-  int8_t     i, syncRequired = 0;
-
-  // pNode->peerInfo[pNode->selfIndex]->version = nodeVersion;
-  pPeer->role = newRole;
+  int8_t oldPeerRole = pPeer->role;
+  int8_t oldSelfRole = nodeRole;
+  int8_t syncRequired = 0;
 
-  sDebug("%s, own role:%s, new peer role:%s", pPeer->id, syncRole[nodeRole], syncRole[pPeer->role]);
+  pPeer->role = newPeerRole;
+  sDebug("%s, peer role:%s change to %s", pPeer->id, syncRole[oldPeerRole], syncRole[newPeerRole]);
 
   SSyncPeer *pMaster = syncCheckMaster(pNode);
 
   if (pMaster) {
     // master is there
     pNode->pMaster = pMaster;
-    sDebug("%s, it is the master, ver:%" PRIu64, pMaster->id, pMaster->version);
+    sDebug("%s, it is the master, sver:%" PRIu64, pMaster->id, pMaster->version);
 
     if (syncValidateMaster(pPeer) < 0) return;
 
     if (nodeRole == TAOS_SYNC_ROLE_UNSYNCED) {
       if (nodeVersion < pMaster->version) {
+        sDebug("%s, is master, sync required, self sver:%" PRIu64, pMaster->id, nodeVersion);
         syncRequired = 1;
       } else {
-        sInfo("%s is master, work as slave, ver:%" PRIu64, pMaster->id, pMaster->version);
+        sInfo("%s, is master, work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion);
         nodeRole = TAOS_SYNC_ROLE_SLAVE;
-        (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+        (*pNode->notifyRole)(pNode->vgId, nodeRole);
       }
     } else if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pMaster == pPeer) {
-      // nodeVersion = pMaster->version;
+      sDebug("%s, is master, continue work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion);
     }
   } else {
     // master not there, if all peer's state and version are consistent, choose the master
     int32_t consistent = 0;
-    if (peersStatus) {
-      for (i = 0; i < pNode->replica; ++i) {
-        SSyncPeer *pTemp = pNode->peerInfo[i];
-        if (pTemp->role != peersStatus[i].role) break;
-        if ((pTemp->role != TAOS_SYNC_ROLE_OFFLINE) && (pTemp->version != peersStatus[i].version)) break;
+    int32_t index = 0;
+    if (peersStatus != NULL) {
+      for (index = 0; index < pNode->replica; ++index) {
+        SSyncPeer *pTemp = pNode->peerInfo[index];
+        if (pTemp->role != peersStatus[index].role) break;
+        if ((pTemp->role != TAOS_SYNC_ROLE_OFFLINE) && (pTemp->version != peersStatus[index].version)) break;
       }
 
-      if (i >= pNode->replica) consistent = 1;
+      if (index >= pNode->replica) consistent = 1;
     } else {
       if (pNode->replica == 2) consistent = 1;
     }
 
     if (consistent) {
+      sDebug("vgId:%d, choose master", pNode->vgId);
       syncChooseMaster(pNode);
+    } else {
+      sDebug("vgId:%d, cannot choose master since roles inequality", pNode->vgId);
     }
   }
 
@@ -728,7 +762,8 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t ne
     syncRecoverFromMaster(pMaster);
   }
 
-  if (peerOldRole != newRole || nodeRole != selfOldRole) {
+  if (oldPeerRole != newPeerRole || nodeRole != oldSelfRole) {
+    sDebug("vgId:%d, roles changed, broadcast status", pNode->vgId);
     syncBroadcastStatus(pNode);
   }
 
@@ -738,14 +773,16 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t ne
 }
 
 static void syncRestartPeer(SSyncPeer *pPeer) {
-  sDebug("%s, restart connection", pPeer->id);
+  sDebug("%s, restart peer connection, last sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
 
   syncClosePeerConn(pPeer);
 
   pPeer->sstatus = TAOS_SYNC_STATUS_INIT;
+  sDebug("%s, peer conn is restart and set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
 
   int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn);
   if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort)) {
+    sDebug("%s, check peer connection in 1000 ms", pPeer->id);
     taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer);
   }
 }
@@ -770,7 +807,7 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) {
   }
 
   if (pPeer->sstatus != TAOS_SYNC_STATUS_INIT) {
-    sDebug("%s, sync is already started", pPeer->id);
+    sDebug("%s, sync is already started for sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
     return;  // already started
   }
 
@@ -784,11 +821,11 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) {
   pthread_attr_destroy(&thattr);
 
   if (ret != 0) {
-    sError("%s, failed to create sync thread(%s)", pPeer->id, strerror(errno));
+    sError("%s, failed to create sync thread since %s", pPeer->id, strerror(errno));
     syncDecPeerRef(pPeer);
   } else {
     pPeer->sstatus = TAOS_SYNC_STATUS_START;
-    sDebug("%s, thread is created to retrieve data", pPeer->id);
+    sDebug("%s, thread is created to retrieve data, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
   }
 }
 
@@ -796,27 +833,28 @@ static void syncNotStarted(void *param, void *tmrId) {
   SSyncPeer *pPeer = param;
   SSyncNode *pNode = pPeer->pSyncNode;
 
-  pthread_mutex_lock(&(pNode->mutex));
+  pthread_mutex_lock(&pNode->mutex);
   pPeer->timer = NULL;
-  sInfo("%s, sync connection is still not up, restart", pPeer->id);
+  pPeer->sstatus = TAOS_SYNC_STATUS_INIT;
+  sInfo("%s, sync conn is still not up, restart and set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
   syncRestartConnection(pPeer);
-  pthread_mutex_unlock(&(pNode->mutex));
+  pthread_mutex_unlock(&pNode->mutex);
 }
 
 static void syncTryRecoverFromMaster(void *param, void *tmrId) {
   SSyncPeer *pPeer = param;
   SSyncNode *pNode = pPeer->pSyncNode;
 
-  pthread_mutex_lock(&(pNode->mutex));
+  pthread_mutex_lock(&pNode->mutex);
   syncRecoverFromMaster(pPeer);
-  pthread_mutex_unlock(&(pNode->mutex));
+  pthread_mutex_unlock(&pNode->mutex);
 }
 
 static void syncRecoverFromMaster(SSyncPeer *pPeer) {
   SSyncNode *pNode = pPeer->pSyncNode;
 
   if (nodeSStatus != TAOS_SYNC_STATUS_INIT) {
-    sDebug("%s, sync is already started, status:%d", pPeer->id, nodeSStatus);
+    sDebug("%s, sync is already started since sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
     return;
   }
 
@@ -840,11 +878,11 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) {
   firstPkt.port = tsSyncPort;
   taosTmrReset(syncNotStarted, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer);
 
-  if (write(pPeer->peerFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) {
+  if (taosWriteMsg(pPeer->peerFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) {
     sError("%s, failed to send sync-req to peer", pPeer->id);
   } else {
     nodeSStatus = TAOS_SYNC_STATUS_START;
-    sInfo("%s, sync-req is sent", pPeer->id);
+    sInfo("%s, sync-req is sent to peer, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
   }
 }
 
@@ -854,7 +892,7 @@ static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) {
   SSyncFwds *pSyncFwds = pNode->pSyncFwds;
   SFwdInfo * pFwdInfo;
 
-  sDebug("%s, forward-rsp is received, code:%x ver:%" PRIu64, pPeer->id, pFwdRsp->code, pFwdRsp->version);
+  sTrace("%s, forward-rsp is received, code:%x hver:%" PRIu64, pPeer->id, pFwdRsp->code, pFwdRsp->version);
   SFwdInfo *pFirst = pSyncFwds->fwdInfo + pSyncFwds->first;
 
   if (pFirst->version <= pFwdRsp->version && pSyncFwds->fwds > 0) {
@@ -873,16 +911,17 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) {
   SSyncNode *pNode = pPeer->pSyncNode;
   SWalHead * pHead = (SWalHead *)cont;
 
-  sDebug("%s, forward is received, hver:%" PRIu64 ", len:%d", pPeer->id, pHead->version, pHead->len);
+  sTrace("%s, forward is received, hver:%" PRIu64 ", len:%d", pPeer->id, pHead->version, pHead->len);
 
   if (nodeRole == TAOS_SYNC_ROLE_SLAVE) {
     // nodeVersion = pHead->version;
-    (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD, NULL);
+    (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL);
   } else {
     if (nodeSStatus != TAOS_SYNC_STATUS_INIT) {
       syncSaveIntoBuffer(pPeer, pHead);
     } else {
-      sError("%s, forward discarded, hver:%" PRIu64, pPeer->id, pHead->version);
+      sError("%s, forward discarded since sstatus:%s, hver:%" PRIu64, pPeer->id, syncStatus[nodeSStatus],
+             pHead->version);
     }
   }
 }
@@ -891,14 +930,15 @@ static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) {
   SSyncNode *   pNode = pPeer->pSyncNode;
   SPeersStatus *pPeersStatus = (SPeersStatus *)cont;
 
-  sDebug("%s, status msg is received, self:%s ver:%" PRIu64 " peer:%s ver:%" PRIu64 ", ack:%d", pPeer->id,
-         syncRole[nodeRole], nodeVersion, syncRole[pPeersStatus->role], pPeersStatus->version, pPeersStatus->ack);
+  sDebug("%s, status is received, self:%s:%s:%" PRIu64 ", peer:%s:%" PRIu64 ", ack:%d tranId:%u type:%s pfd:%d",
+         pPeer->id, syncRole[nodeRole], syncStatus[nodeSStatus], nodeVersion, syncRole[pPeersStatus->role],
+         pPeersStatus->version, pPeersStatus->ack, pPeersStatus->tranId, statusType[pPeersStatus->type], pPeer->peerFd);
 
   pPeer->version = pPeersStatus->version;
   syncCheckRole(pPeer, pPeersStatus->peersStatus, pPeersStatus->role);
 
   if (pPeersStatus->ack) {
-    syncSendPeersStatusMsgToPeer(pPeer, 0);
+    syncSendPeersStatusMsgToPeer(pPeer, 0, pPeersStatus->type + 1, pPeersStatus->tranId);
   }
 }
 
@@ -933,7 +973,7 @@ static int32_t syncProcessPeerMsg(void *param, void *buffer) {
   char *     cont = buffer;
 
   SSyncNode *pNode = pPeer->pSyncNode;
-  pthread_mutex_lock(&(pNode->mutex));
+  pthread_mutex_lock(&pNode->mutex);
 
   int32_t code = syncReadPeerMsg(pPeer, &head, cont);
 
@@ -949,14 +989,14 @@ static int32_t syncProcessPeerMsg(void *param, void *buffer) {
     }
   }
 
-  pthread_mutex_unlock(&(pNode->mutex));
+  pthread_mutex_unlock(&pNode->mutex);
 
   return code;
 }
 
 #define statusMsgLen sizeof(SSyncHead) + sizeof(SPeersStatus) + sizeof(SPeerStatus) * TAOS_SYNC_MAX_REPLICA
 
-static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack) {
+static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId) {
   SSyncNode *pNode = pPeer->pSyncNode;
   char       msg[statusMsgLen] = {0};
 
@@ -971,22 +1011,24 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack) {
   pPeersStatus->version = nodeVersion;
   pPeersStatus->role = nodeRole;
   pPeersStatus->ack = ack;
+  pPeersStatus->type = type;
+  pPeersStatus->tranId = tranId;
 
   for (int32_t i = 0; i < pNode->replica; ++i) {
     pPeersStatus->peersStatus[i].role = pNode->peerInfo[i]->role;
     pPeersStatus->peersStatus[i].version = pNode->peerInfo[i]->version;
   }
 
-  int32_t retLen = write(pPeer->peerFd, msg, statusMsgLen);
+  int32_t retLen = taosWriteMsg(pPeer->peerFd, msg, statusMsgLen);
   if (retLen == statusMsgLen) {
-    sDebug("%s, status msg is sent, self:%s ver:%" PRIu64 ", ack:%d", pPeer->id, syncRole[pPeersStatus->role],
-           pPeersStatus->version, pPeersStatus->ack);
+    sDebug("%s, status is sent, self:%s:%s:%" PRIu64 ", peer:%s:%s:%" PRIu64 ", ack:%d tranId:%u type:%s pfd:%d",
+           pPeer->id, syncRole[nodeRole], syncStatus[nodeSStatus], nodeVersion, syncRole[pPeer->role],
+           syncStatus[pPeer->sstatus], pPeer->version, pPeersStatus->ack, pPeersStatus->tranId,
+           statusType[pPeersStatus->type], pPeer->peerFd);
   } else {
     sDebug("%s, failed to send status msg, restart", pPeer->id);
     syncRestartConnection(pPeer);
   }
-
-  return;
 }
 
 static void syncSetupPeerConnection(SSyncPeer *pPeer) {
@@ -995,13 +1037,13 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) {
   taosTmrStopA(&pPeer->timer);
   if (pPeer->peerFd >= 0) {
     sDebug("%s, send role version to peer", pPeer->id);
-    syncSendPeersStatusMsgToPeer(pPeer, 1);
+    syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_SETUP_CONN, syncGenTranId());
     return;
   }
 
   int32_t connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0);
   if (connFd < 0) {
-    sDebug("%s, failed to open tcp socket(%s)", pPeer->id, strerror(errno));
+    sDebug("%s, failed to open tcp socket since %s", pPeer->id, strerror(errno));
     taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer);
     return;
   }
@@ -1014,15 +1056,15 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) {
   firstPkt.port = tsSyncPort;
   firstPkt.sourceId = pNode->vgId;  // tell arbitrator its vgId
 
-  if (write(connFd, &firstPkt, sizeof(firstPkt)) == sizeof(firstPkt)) {
-    sDebug("%s, connection to peer server is setup", pPeer->id);
+  if (taosWriteMsg(connFd, &firstPkt, sizeof(firstPkt)) == sizeof(firstPkt)) {
+    sDebug("%s, connection to peer server is setup, pfd:%d sfd:%d", pPeer->id, connFd, pPeer->syncFd);
     pPeer->peerFd = connFd;
     pPeer->role = TAOS_SYNC_ROLE_UNSYNCED;
     pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd);
     syncAddPeerRef(pPeer);
   } else {
-    sDebug("try later");
-    close(connFd);
+    sDebug("%s, failed to setup peer connection to server since %s, try later", pPeer->id, strerror(errno));
+    taosClose(connFd);
     taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer);
   }
 }
@@ -1031,12 +1073,12 @@ static void syncCheckPeerConnection(void *param, void *tmrId) {
   SSyncPeer *pPeer = param;
   SSyncNode *pNode = pPeer->pSyncNode;
 
-  pthread_mutex_lock(&(pNode->mutex));
+  pthread_mutex_lock(&pNode->mutex);
 
   sDebug("%s, check peer connection", pPeer->id);
   syncSetupPeerConnection(pPeer);
 
-  pthread_mutex_unlock(&(pNode->mutex));
+  pthread_mutex_unlock(&pNode->mutex);
 }
 
 static void syncCreateRestoreDataThread(SSyncPeer *pPeer) {
@@ -1069,13 +1111,13 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
 
   SFirstPkt firstPkt;
   if (taosReadMsg(connFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) {
-    sError("failed to read peer first pkt from ip:%s(%s)", ipstr, strerror(errno));
+    sError("failed to read peer first pkt from ip:%s since %s", ipstr, strerror(errno));
     taosCloseSocket(connFd);
     return;
   }
 
   int32_t     vgId = firstPkt.syncHead.vgId;
-  SSyncNode **ppNode = (SSyncNode **)taosHashGet(tsVgIdHash, (const char *)&vgId, sizeof(int32_t));
+  SSyncNode **ppNode = taosHashGet(tsVgIdHash, &vgId, sizeof(int32_t));
   if (ppNode == NULL || *ppNode == NULL) {
     sError("vgId:%d, vgId could not be found", vgId);
     taosCloseSocket(connFd);
@@ -1083,7 +1125,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
   }
 
   SSyncNode *pNode = *ppNode;
-  pthread_mutex_lock(&(pNode->mutex));
+  pthread_mutex_lock(&pNode->mutex);
 
   SSyncPeer *pPeer;
   for (i = 0; i < pNode->replica; ++i) {
@@ -1102,17 +1144,17 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
       pPeer->syncFd = connFd;
       syncCreateRestoreDataThread(pPeer);
     } else {
-      sDebug("%s, TCP connection is already up, close one", pPeer->id);
+      sDebug("%s, TCP connection is up, pfd:%d sfd:%d, old pfd:%d", pPeer->id, connFd, pPeer->syncFd, pPeer->peerFd);
       syncClosePeerConn(pPeer);
       pPeer->peerFd = connFd;
       pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd);
       syncAddPeerRef(pPeer);
       sDebug("%s, ready to exchange data", pPeer->id);
-      syncSendPeersStatusMsgToPeer(pPeer, 1);
+      syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_EXCHANGE_DATA, syncGenTranId());
     }
   }
 
-  pthread_mutex_unlock(&(pNode->mutex));
+  pthread_mutex_unlock(&pNode->mutex);
 }
 
 static void syncProcessBrokenLink(void *param) {
@@ -1121,22 +1163,22 @@ static void syncProcessBrokenLink(void *param) {
   SSyncNode *pNode = pPeer->pSyncNode;
 
   if (taosAcquireRef(tsSyncRefId, pNode->rid) == NULL) return;
-  pthread_mutex_lock(&(pNode->mutex));
+  pthread_mutex_lock(&pNode->mutex);
 
-  sDebug("%s, TCP link is broken(%s)", pPeer->id, strerror(errno));
+  sDebug("%s, TCP link is broken since %s, pfd:%d sfd:%d", pPeer->id, strerror(errno), pPeer->peerFd, pPeer->syncFd);
   pPeer->peerFd = -1;
 
   if (syncDecPeerRef(pPeer) != 0) {
     syncRestartConnection(pPeer);
   }
 
-  pthread_mutex_unlock(&(pNode->mutex));
+  pthread_mutex_unlock(&pNode->mutex);
   taosReleaseRef(tsSyncRefId, pNode->rid);
 }
 
 static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) {
   SSyncFwds *pSyncFwds = pNode->pSyncFwds;
-  uint64_t   time = taosGetTimestampMs();
+  int64_t    time = taosGetTimestampMs();
 
   if (pSyncFwds->fwds >= tsMaxFwdInfo) {
     pSyncFwds->first = (pSyncFwds->first + 1) % tsMaxFwdInfo;
@@ -1154,7 +1196,7 @@ static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) {
   pFwdInfo->time = time;
 
   pSyncFwds->fwds++;
-  sDebug("vgId:%d, fwd info is saved, ver:%" PRIu64 " fwds:%d ", pNode->vgId, version, pSyncFwds->fwds);
+  sTrace("vgId:%d, fwd info is saved, hver:%" PRIu64 " fwds:%d ", pNode->vgId, version, pSyncFwds->fwds);
 }
 
 static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) {
@@ -1168,7 +1210,7 @@ static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) {
     pSyncFwds->first = (pSyncFwds->first + 1) % tsMaxFwdInfo;
     pSyncFwds->fwds--;
     if (pSyncFwds->fwds == 0) pSyncFwds->first = pSyncFwds->last;
-    // sDebug("vgId:%d, fwd info is removed, ver:%d, fwds:%d",
+    // sDebug("vgId:%d, fwd info is removed, hver:%d, fwds:%d",
     //        pNode->vgId, pFwdInfo->version, pSyncFwds->fwds);
     memset(pFwdInfo, 0, sizeof(SFwdInfo));
   }
@@ -1191,14 +1233,36 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code
   }
 
   if (confirm && pFwdInfo->confirmed == 0) {
-    sDebug("vgId:%d, forward is confirmed, ver:%" PRIu64 " code:%x", pNode->vgId, pFwdInfo->version, pFwdInfo->code);
-    (*pNode->confirmForward)(pNode->ahandle, pFwdInfo->mhandle, pFwdInfo->code);
+    sTrace("vgId:%d, forward is confirmed, hver:%" PRIu64 " code:%x", pNode->vgId, pFwdInfo->version, pFwdInfo->code);
+    (*pNode->confirmForward)(pNode->vgId, pFwdInfo->mhandle, pFwdInfo->code);
     pFwdInfo->confirmed = 1;
   }
 }
 
+static void syncMonitorNodeRole(void *param, void *tmrId) {
+  int64_t    rid = (int64_t)param;
+  SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+  if (pNode == NULL) return;
+
+  for (int32_t index = 0; index < pNode->replica; index++) {
+    if (index == pNode->selfIndex) continue;
+
+    SSyncPeer *pPeer = pNode->peerInfo[index];
+    if (/*pPeer->role > TAOS_SYNC_ROLE_UNSYNCED && */ nodeRole > TAOS_SYNC_ROLE_UNSYNCED) continue;
+    if (/*pPeer->sstatus > TAOS_SYNC_STATUS_INIT || */ nodeSStatus > TAOS_SYNC_STATUS_INIT) continue;
+
+    sDebug("%s, check roles since self:%s sstatus:%s, peer:%s sstatus:%s", pPeer->id, syncRole[pPeer->role],
+           syncStatus[pPeer->sstatus], syncRole[nodeRole], syncStatus[nodeSStatus]);
+    syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_CHECK_ROLE, syncGenTranId());
+    break;
+  }
+
+  pNode->pRoleTimer = taosTmrStart(syncMonitorNodeRole, SYNC_ROLE_TIMER, (void *)pNode->rid, tsSyncTmrCtrl);
+  taosReleaseRef(tsSyncRefId, rid);
+}
+
 static void syncMonitorFwdInfos(void *param, void *tmrId) {
-  int64_t rid = (int64_t) param;
+  int64_t    rid = (int64_t)param;
   SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
   if (pNode == NULL) return;
 
@@ -1208,21 +1272,21 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) {
     int64_t time = taosGetTimestampMs();
 
     if (pSyncFwds->fwds > 0) {
-      pthread_mutex_lock(&(pNode->mutex));
+      pthread_mutex_lock(&pNode->mutex);
       for (int32_t i = 0; i < pSyncFwds->fwds; ++i) {
         SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % tsMaxFwdInfo;
         if (ABS(time - pFwdInfo->time) < 2000) break;
 
         sDebug("vgId:%d, forward info expired, hver:%" PRIu64 " curtime:%" PRIu64 " savetime:%" PRIu64, pNode->vgId,
                pFwdInfo->version, time, pFwdInfo->time);
-        syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_RPC_NETWORK_UNAVAIL);
+        syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_SYN_CONFIRM_EXPIRED);
       }
 
       syncRemoveConfirmedFwdInfo(pNode);
-      pthread_mutex_unlock(&(pNode->mutex));
+      pthread_mutex_unlock(&pNode->mutex);
     }
 
-    pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, tsSyncTmrCtrl);
+    pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, SYNC_FWD_TIMER, (void *)pNode->rid, tsSyncTmrCtrl);
   }
 
   taosReleaseRef(tsSyncRefId, rid);
@@ -1235,9 +1299,8 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle
   int32_t    fwdLen;
   int32_t    code = 0;
 
-
   if (pWalHead->version > nodeVersion + 1) {
-    sError("vgId:%d, hver:%" PRIu64 ", inconsistent with ver:%" PRIu64, pNode->vgId, pWalHead->version, nodeVersion);
+    sError("vgId:%d, hver:%" PRIu64 ", inconsistent with sver:%" PRIu64, pNode->vgId, pWalHead->version, nodeVersion);
     if (nodeRole == TAOS_SYNC_ROLE_SLAVE) {
       sInfo("vgId:%d, restart connection", pNode->vgId);
       for (int32_t i = 0; i < pNode->replica; ++i) {
@@ -1250,9 +1313,9 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle
   }
 
   // always update version
-  nodeVersion = pWalHead->version;
   sTrace("vgId:%d, forward to peer, replica:%d role:%s qtype:%s hver:%" PRIu64, pNode->vgId, pNode->replica,
          syncRole[nodeRole], qtypeStr[qtype], pWalHead->version);
+  nodeVersion = pWalHead->version;
 
   if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0;
 
@@ -1266,7 +1329,7 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle
   pSyncHead->len = sizeof(SWalHead) + pWalHead->len;
   fwdLen = pSyncHead->len + sizeof(SSyncHead);  // include the WAL and SYNC head
 
-  pthread_mutex_lock(&(pNode->mutex));
+  pthread_mutex_lock(&pNode->mutex);
 
   for (int32_t i = 0; i < pNode->replica; ++i) {
     pPeer = pNode->peerInfo[i];
@@ -1278,16 +1341,18 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle
       code = 1;
     }
 
-    int32_t retLen = write(pPeer->peerFd, pSyncHead, fwdLen);
+    int32_t retLen = taosWriteMsg(pPeer->peerFd, pSyncHead, fwdLen);
     if (retLen == fwdLen) {
-      sDebug("%s, forward is sent, ver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len);
+      sTrace("%s, forward is sent, role:%s sstatus:%s hver:%" PRIu64 " contLen:%d", pPeer->id, syncRole[pPeer->role],
+             syncStatus[pPeer->sstatus], pWalHead->version, pWalHead->len);
     } else {
-      sError("%s, failed to forward, ver:%" PRIu64 " retLen:%d", pPeer->id, pWalHead->version, retLen);
+      sError("%s, failed to forward, role:%s sstatus:%s hver:%" PRIu64 " retLen:%d", pPeer->id, syncRole[pPeer->role],
+             syncStatus[pPeer->sstatus], pWalHead->version, retLen);
       syncRestartConnection(pPeer);
     }
   }
 
-  pthread_mutex_unlock(&(pNode->mutex));
+  pthread_mutex_unlock(&pNode->mutex);
 
   return code;
 }

src/sync/src/syncRestore.c

@@ -13,7 +13,9 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define _DEFAULT_SOURCE
 #include "os.h"
+#include "taoserror.h"
 #include "tlog.h"
 #include "tutil.h"
 #include "ttimer.h"
@@ -36,7 +38,7 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex
 
   while (1) {
     name[0] = 0;
-    magic = (*pNode->getFileInfo)(pNode->ahandle, name, &index, eindex, &size, &fversion);
+    magic = (*pNode->getFileInfo)(pNode->vgId, name, &index, eindex, &size, &fversion);
     if (magic == 0) break;
 
     snprintf(fname, sizeof(fname), "%s/%s", pNode->path, name);
@@ -50,9 +52,9 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex
 
 static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
   SSyncNode *pNode = pPeer->pSyncNode;
-  SFileInfo  minfo; memset(&minfo, 0, sizeof(minfo)); /* = {0}; */  // master file info
-  SFileInfo  sinfo; memset(&sinfo, 0, sizeof(sinfo)); /* = {0}; */  // slave file info
-  SFileAck   fileAck; 
+  SFileInfo  minfo; memset(&minfo, 0, sizeof(SFileInfo)); /* = {0}; */
+  SFileInfo  sinfo; memset(&sinfo, 0, sizeof(SFileInfo)); /* = {0}; */
+  SFileAck   fileAck = {0};
   int32_t    code = -1;
   char       name[TSDB_FILENAME_LEN * 2] = {0};
   uint32_t   pindex = 0;  // index in last restore
@@ -63,7 +65,10 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
   while (1) {
     // read file info
     int32_t ret = taosReadMsg(pPeer->syncFd, &(minfo), sizeof(minfo));
-    if (ret < 0) break;
+    if (ret < 0) {
+      sError("%s, failed to read file info while restore file since %s", pPeer->id, strerror(errno));
+      break;
+    }
 
     // if no more file from master, break;
     if (minfo.name[0] == 0 || minfo.magic == 0) {
@@ -81,8 +86,8 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
 
     // check the file info
     sinfo = minfo;
-    sDebug("%s, get file info:%s", pPeer->id, minfo.name);
-    sinfo.magic = (*pNode->getFileInfo)(pNode->ahandle, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size,
+    sDebug("%s, get file:%s info size:%" PRId64, pPeer->id, minfo.name, minfo.size);
+    sinfo.magic = (*pNode->getFileInfo)(pNode->vgId, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size,
                                         &sinfo.fversion);
 
     // if file not there or magic is not the same, file shall be synced
@@ -90,8 +95,11 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
     fileAck.sync = (sinfo.magic != minfo.magic || sinfo.name[0] == 0) ? 1 : 0;
 
     // send file ack
-    ret = taosWriteMsg(pPeer->syncFd, &(fileAck), sizeof(fileAck));
-    if (ret < 0) break;
+    ret = taosWriteMsg(pPeer->syncFd, &fileAck, sizeof(fileAck));
+    if (ret < 0) {
+      sError("%s, failed to write file:%s ack while restore file since %s", pPeer->id, minfo.name, strerror(errno));
+      break;
+    }
 
     // if sync is not required, continue
     if (fileAck.sync == 0) {
@@ -106,14 +114,17 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
 
     int32_t dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
     if (dfd < 0) {
-      sError("%s, failed to open file:%s", pPeer->id, name);
+      sError("%s, failed to open file:%s while restore file since %s", pPeer->id, minfo.name, strerror(errno));
       break;
     }
 
     ret = taosCopyFds(pPeer->syncFd, dfd, minfo.size);
     fsync(dfd);
     close(dfd);
-    if (ret < 0) break;
+    if (ret < 0) {
+      sError("%s, failed to copy file:%s while restore file since %s", pPeer->id, minfo.name, strerror(errno));
+      break;
+    }
 
     fileChanged = true;
     sDebug("%s, %s is received, size:%" PRId64, pPeer->id, minfo.name, minfo.size);
@@ -123,10 +134,11 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
     // data file is changed, code shall be set to 1
     *fversion = minfo.fversion;
     code = 1;
+    sDebug("%s, file changed after restore file, fver:%" PRIu64, pPeer->id, *fversion);
   }
 
   if (code < 0) {
-    sError("%s, failed to restore %s(%s)", pPeer->id, name, strerror(errno));
+    sError("%s, failed to restore %s since %s", pPeer->id, name, strerror(errno));
   }
 
   return code;
@@ -135,33 +147,46 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
 static int32_t syncRestoreWal(SSyncPeer *pPeer) {
   SSyncNode *pNode = pPeer->pSyncNode;
   int32_t    ret, code = -1;
+  uint64_t   lastVer = 0;
 
-  void *buffer = calloc(SYNC_MAX_SIZE, 1);  // size for one record
-  if (buffer == NULL) return -1;
-
-  SWalHead *pHead = (SWalHead *)buffer;
+  SWalHead *pHead = calloc(SYNC_MAX_SIZE, 1);  // size for one record
+  if (pHead == NULL) return -1;
 
   while (1) {
     ret = taosReadMsg(pPeer->syncFd, pHead, sizeof(SWalHead));
-    if (ret < 0) break;
+    if (ret < 0) {
+      sError("%s, failed to read walhead while restore wal since %s", pPeer->id, strerror(errno));
+      break;
+    }
 
     if (pHead->len == 0) {
+      sDebug("%s, wal is synced over, last wver:%" PRIu64, pPeer->id, lastVer);
       code = 0;
       break;
     }  // wal sync over
 
     ret = taosReadMsg(pPeer->syncFd, pHead->cont, pHead->len);
-    if (ret < 0) break;
+    if (ret < 0) {
+      sError("%s, failed to read walcont, len:%d while restore wal since %s", pPeer->id, pHead->len, strerror(errno));
+      break;
+    }
+
+    sTrace("%s, restore a record, qtype:wal len:%d hver:%" PRIu64, pPeer->id, pHead->len, pHead->version);
 
-    sDebug("%s, restore a record, qtype:wal hver:%" PRIu64, pPeer->id, pHead->version);
-    (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_WAL, NULL);
+    if (lastVer == pHead->version) {
+      sError("%s, failed to restore record, same hver:%" PRIu64 ", wal sync failed" PRIu64, pPeer->id, lastVer);
+      break;
+    }
+    lastVer = pHead->version;
+
+    (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_WAL, NULL);
   }
 
   if (code < 0) {
-    sError("%s, failed to restore wal(%s)", pPeer->id, strerror(errno));
+    sError("%s, failed to restore wal from syncFd:%d since %s", pPeer->id, pPeer->syncFd, strerror(errno));
   }
 
-  free(buffer);
+  free(pHead);
   return code;
 }
 
@@ -169,7 +194,7 @@ static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) {
   SSyncNode *pNode = pPeer->pSyncNode;
   SWalHead * pHead = (SWalHead *)offset;
 
-  (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD, NULL);
+  (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL);
   offset += pHead->len + sizeof(SWalHead);
 
   return offset;
@@ -206,15 +231,18 @@ static int32_t syncProcessBufferedFwd(SSyncPeer *pPeer) {
 int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) {
   SSyncNode *  pNode = pPeer->pSyncNode;
   SRecvBuffer *pRecv = pNode->pRecv;
-
-  if (pRecv == NULL) return -1;
   int32_t len = pHead->len + sizeof(SWalHead);
 
+  if (pRecv == NULL) {
+    sError("%s, recv buffer is not create yet", pPeer->id);
+    return -1;
+  }
+
   if (pRecv->bufferSize - (pRecv->offset - pRecv->buffer) >= len) {
     memcpy(pRecv->offset, pHead, len);
     pRecv->offset += len;
     pRecv->forwards++;
-    sDebug("%s, fwd is saved into queue, ver:%" PRIu64 " fwds:%d", pPeer->id, pHead->version, pRecv->forwards);
+    sTrace("%s, fwd is saved into queue, hver:%" PRIu64 " fwds:%d", pPeer->id, pHead->version, pRecv->forwards);
   } else {
     sError("%s, buffer size:%d is too small", pPeer->id, pRecv->bufferSize);
     pRecv->code = -1;  // set error code
@@ -257,7 +285,14 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) {
   nodeSStatus = TAOS_SYNC_STATUS_FILE;
   uint64_t fversion = 0;
 
-  sDebug("%s, start to restore file", pPeer->id);
+  sInfo("%s, start to restore, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
+  SFirstPktRsp firstPktRsp = {.sync = 1};
+  if (taosWriteMsg(pPeer->syncFd, &firstPktRsp, sizeof(SFirstPktRsp)) < 0) {
+    sError("%s, failed to send sync firstPkt rsp since %s", pPeer->id, strerror(errno));
+    return -1;
+  }
+
+  sInfo("%s, start to restore file, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
   int32_t code = syncRestoreFile(pPeer, &fversion);
   if (code < 0) {
     sError("%s, failed to restore file", pPeer->id);
@@ -266,7 +301,7 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) {
 
   // if code > 0, data file is changed, notify app, and pass the version
   if (code > 0 && pNode->notifyFileSynced) {
-    if ((*pNode->notifyFileSynced)(pNode->ahandle, fversion) < 0) {
+    if ((*pNode->notifyFileSynced)(pNode->vgId, fversion) < 0) {
       sError("%s, app not in ready state", pPeer->id);
       return -1;
     }
@@ -274,14 +309,14 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) {
 
   nodeVersion = fversion;
 
-  sDebug("%s, start to restore wal", pPeer->id);
+  sInfo("%s, start to restore wal", pPeer->id);
   if (syncRestoreWal(pPeer) < 0) {
     sError("%s, failed to restore wal", pPeer->id);
     return -1;
   }
 
   nodeSStatus = TAOS_SYNC_STATUS_CACHE;
-  sDebug("%s, start to insert buffered points", pPeer->id);
+  sInfo("%s, start to insert buffered points, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
   if (syncProcessBufferedFwd(pPeer) < 0) {
     sError("%s, failed to insert buffered points", pPeer->id);
     return -1;
@@ -291,16 +326,17 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) {
 }
 
 void *syncRestoreData(void *param) {
-  SSyncPeer *pPeer = (SSyncPeer *)param;
+  SSyncPeer *pPeer = param;
   SSyncNode *pNode = pPeer->pSyncNode;
 
   taosBlockSIGPIPE();
   __sync_fetch_and_add(&tsSyncNum, 1);
 
-  (*pNode->notifyRole)(pNode->ahandle, TAOS_SYNC_ROLE_SYNCING);
+  (*pNode->notifyRole)(pNode->vgId, TAOS_SYNC_ROLE_SYNCING);
 
   if (syncOpenRecvBuffer(pNode) < 0) {
-    sError("%s, failed to allocate recv buffer", pPeer->id);
+    sError("%s, failed to allocate recv buffer, restart connection", pPeer->id);
+    syncRestartConnection(pPeer);
   } else {
     if (syncRestoreDataStepByStep(pPeer) == 0) {
       sInfo("%s, it is synced successfully", pPeer->id);
@@ -313,9 +349,11 @@ void *syncRestoreData(void *param) {
     }
   }
 
-  (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+  (*pNode->notifyRole)(pNode->vgId, nodeRole);
 
   nodeSStatus = TAOS_SYNC_STATUS_INIT;
+  sInfo("%s, sync over, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
+
   taosClose(pPeer->syncFd);
   syncCloseRecvBuffer(pNode);
   __sync_fetch_and_sub(&tsSyncNum, 1);

src/sync/src/syncRetrieve.c

@@ -13,11 +13,10 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdint.h>
-#include <stdbool.h>
+#define _DEFAULT_SOURCE
 #include <sys/inotify.h>
-#include <unistd.h>
 #include "os.h"
+#include "taoserror.h"
 #include "tlog.h"
 #include "tutil.h"
 #include "tglobal.h"
@@ -27,117 +26,70 @@
 #include "tsync.h"
 #include "syncInt.h"
 
-static int32_t syncAddIntoWatchList(SSyncPeer *pPeer, char *name) {
-  sDebug("%s, start to monitor:%s", pPeer->id, name);
+static int32_t syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) {
+  if (pNode->getFileVersion == NULL) return TSDB_CODE_SUCCESS;
 
-  if (pPeer->notifyFd <= 0) {
-    pPeer->watchNum = 0;
-    pPeer->notifyFd = inotify_init1(IN_NONBLOCK);
-    if (pPeer->notifyFd < 0) {
-      sError("%s, failed to init inotify(%s)", pPeer->id, strerror(errno));
-      return -1;
-    }
-
-    if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int32_t) * tsMaxWatchFiles);
-    if (pPeer->watchFd == NULL) {
-      sError("%s, failed to allocate watchFd", pPeer->id);
-      return -1;
-    }
-
-    memset(pPeer->watchFd, -1, sizeof(int32_t) * tsMaxWatchFiles);
-  }
-
-  int32_t *wd = pPeer->watchFd + pPeer->watchNum;
-
-  if (*wd >= 0) {
-    if (inotify_rm_watch(pPeer->notifyFd, *wd) < 0) {
-      sError("%s, failed to remove wd:%d(%s)", pPeer->id, *wd, strerror(errno));
-      return -1;
-    }
-  }
-
-  *wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY | IN_DELETE);
-  if (*wd == -1) {
-    sError("%s, failed to add %s(%s)", pPeer->id, name, strerror(errno));
-    return -1;
-  } else {
-    sDebug("%s, monitor %s, wd:%d watchNum:%d", pPeer->id, name, *wd, pPeer->watchNum);
-  }
-
-  pPeer->watchNum = (pPeer->watchNum + 1) % tsMaxWatchFiles;
-
-  return 0;
-}
-
-static int32_t syncAreFilesModified(SSyncPeer *pPeer) {
-  if (pPeer->notifyFd <= 0) return 0;
-
-  char    buf[2048];
-  int32_t len = read(pPeer->notifyFd, buf, sizeof(buf));
-  if (len < 0 && errno != EAGAIN) {
-    sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno));
-    return -1;
+  uint64_t fver = 0;
+  int32_t  code = (*pNode->getFileVersion)(pNode->vgId, &fver);
+  if (code != 0) {
+    sInfo("%s, vnode is commiting while retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastVer);
+    pPeer->fileChanged = 1;
+    return TSDB_CODE_SYN_VND_COMMITING;
   }
 
-  int32_t code = 0;
-  if (len > 0) {
-    const struct inotify_event *event;
-    char *ptr;
-    for (ptr = buf; ptr < buf + len; ptr += sizeof(struct inotify_event) + event->len) {
-      event = (const struct inotify_event *)ptr;
-      if ((event->mask & IN_MODIFY) || (event->mask & IN_DELETE)) {
-        sDebug("%s, processed file is changed", pPeer->id);
+  if (fver != pPeer->lastVer) {
+    sInfo("%s, files are modified while retrieve, fver:%" PRIu64 ", last fver:%" PRIu64, pPeer->id, fver, pPeer->lastVer);
     pPeer->fileChanged = 1;
-        code = 1;
-        break;
-      }
-    }
+    return TSDB_CODE_SYN_FILE_CHNAGED;
   }
 
-  return code;
+  pPeer->fileChanged = 0;
+  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
   SSyncNode *pNode = pPeer->pSyncNode;
-  SFileInfo  fileInfo;
-  SFileAck   fileAck;
-  int32_t    code = -1;
+  SFileInfo  fileInfo; memset(&fileInfo, 0, sizeof(SFileInfo));
+  SFileAck   fileAck = {0};
+  int32_t    code = TSDB_CODE_SYN_APP_ERROR;
   char       name[TSDB_FILENAME_LEN * 2] = {0};
 
-  memset(&fileInfo, 0, sizeof(fileInfo));
-  memset(&fileAck, 0, sizeof(fileAck));
+  if (pNode->getFileVersion) (*pNode->getFileVersion)(pNode->vgId, &pPeer->lastVer);
 
   while (1) {
     // retrieve file info
     fileInfo.name[0] = 0;
-    fileInfo.magic = (*pNode->getFileInfo)(pNode->ahandle, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX,
+    fileInfo.magic = (*pNode->getFileInfo)(pNode->vgId, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX,
                                            &fileInfo.size, &fileInfo.fversion);
     // fileInfo.size = htonl(size);
+    sDebug("%s, file:%s info is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size);
 
     // send the file info
     int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(fileInfo));
-    if (ret < 0) break;
+    if (ret < 0) {
+      code = TAOS_SYSTEM_ERROR(errno);
+      sError("%s, failed to write file:%s info while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
+      break;
+    }
 
     // if no file anymore, break
     if (fileInfo.magic == 0 || fileInfo.name[0] == 0) {
+      code = TSDB_CODE_SUCCESS;
       sDebug("%s, no more files to sync", pPeer->id);
-      code = 0;
       break;
     }
 
     // wait for the ack from peer
-    ret = taosReadMsg(pPeer->syncFd, &(fileAck), sizeof(fileAck));
-    if (ret < 0) break;
+    ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(fileAck));
+    if (ret < 0) {
+      code = TAOS_SYSTEM_ERROR(errno);
+      sError("%s, failed to read file:%s ack while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
+      break;
+    }
 
     // set the peer sync version
     pPeer->sversion = fileInfo.fversion;
 
-    // get the full path to file
-    snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name);
-
-    // add the file into watch list
-    if (syncAddIntoWatchList(pPeer, name) < 0) break;
-
     // if sync is not required, continue
     if (fileAck.sync == 0) {
       fileInfo.index++;
@@ -145,23 +97,35 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
       continue;
     }
 
+    // get the full path to file
+    snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name);
+
     // send the file to peer
     int32_t sfd = open(name, O_RDONLY);
-    if (sfd < 0) break;
+    if (sfd < 0) {
+      code = TAOS_SYSTEM_ERROR(errno);
+      sError("%s, failed to open file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
+      break;
+    }
 
     ret = taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size);
     close(sfd);
-    if (ret < 0) break;
+    if (ret < 0) {
+      code = TAOS_SYSTEM_ERROR(errno);
+      sError("%s, failed to send file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
+      break;
+    }
 
-    sDebug("%s, %s is sent, size:%" PRId64, pPeer->id, name, fileInfo.size);
+    sDebug("%s, file:%s is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size);
     fileInfo.index++;
 
     // check if processed files are modified
-    if (syncAreFilesModified(pPeer) != 0) break;
+    code = syncAreFilesModified(pNode, pPeer);
+    if (code != TSDB_CODE_SUCCESS) break;
   }
 
-  if (code < 0) {
-    sError("%s, failed to retrieve file(%s)", pPeer->id, strerror(errno));
+  if (code != TSDB_CODE_SUCCESS) {
+    sError("%s, failed to retrieve file since %s", pPeer->id, tstrerror(code));
   }
 
   return code;
@@ -201,7 +165,7 @@ static int32_t syncMonitorLastWal(SSyncPeer *pPeer, char *name) {
   taosClose(pPeer->notifyFd);
   pPeer->notifyFd = inotify_init1(IN_NONBLOCK);
   if (pPeer->notifyFd < 0) {
-    sError("%s, failed to init inotify(%s)", pPeer->id, strerror(errno));
+    sError("%s, failed to init inotify since %s", pPeer->id, strerror(errno));
     return -1;
   }
 
@@ -216,7 +180,7 @@ static int32_t syncMonitorLastWal(SSyncPeer *pPeer, char *name) {
 
   *wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY | IN_CLOSE_WRITE);
   if (*wd == -1) {
-    sError("%s, failed to watch last wal(%s)", pPeer->id, strerror(errno));
+    sError("%s, failed to watch last wal since %s", pPeer->id, strerror(errno));
     return -1;
   }
 
@@ -227,7 +191,7 @@ static int32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) {
   char    buf[2048];
   int32_t len = read(pPeer->notifyFd, buf, sizeof(buf));
   if (len < 0 && errno != EAGAIN) {
-    sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno));
+    sError("%s, failed to read notify FD since %s", pPeer->id, strerror(errno));
     return -1;
   }
 
@@ -268,7 +232,7 @@ static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversi
       break;
     }
 
-    sDebug("%s, last wal is forwarded, ver:%" PRIu64, pPeer->id, pHead->version);
+    sTrace("%s, last wal is forwarded, hver:%" PRIu64, pPeer->id, pHead->version);
     int32_t ret = taosWriteMsg(pPeer->syncFd, pHead, wsize);
     if (ret != wsize) break;
     pPeer->sversion = pHead->version;
@@ -292,9 +256,9 @@ static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversi
 static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) {
   SSyncNode *pNode = pPeer->pSyncNode;
   int32_t    code = -1;
-  char       fname[TSDB_FILENAME_LEN * 2];  // full path to wal file
+  char       fname[TSDB_FILENAME_LEN * 2] = {0};  // full path to wal file
 
-  if (syncAreFilesModified(pPeer) != 0) return -1;
+  if (syncAreFilesModified(pNode, pPeer) != 0) return -1;
 
   while (1) {
     int32_t  once = 0;  // last WAL has once ever been processed
@@ -320,6 +284,7 @@ static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index)
       if (((event & IN_MODIFY) == 0) || once) {
         if (fversion == 0) {
           pPeer->sstatus = TAOS_SYNC_STATUS_CACHE;  // start to forward pkt
+          sDebug("%s, fversion is 0 then set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
           fversion = nodeVersion;                   // must read data to fversion
         }
       }
@@ -356,7 +321,7 @@ static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index)
 
     index++;
     wname[0] = 0;
-    code = (*pNode->getWalInfo)(pNode->ahandle, wname, &index);
+    code = (*pNode->getWalInfo)(pNode->vgId, wname, &index);
     if (code < 0) break;
     if (wname[0] == 0) {
       code = 0;
@@ -384,7 +349,7 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) {
   while (1) {
     // retrieve wal info
     wname[0] = 0;
-    code = (*pNode->getWalInfo)(pNode->ahandle, wname, &index);
+    code = (*pNode->getWalInfo)(pNode->vgId, wname, &index);
     if (code < 0) break;  // error
     if (wname[0] == 0) {  // no wal file
       sDebug("%s, no wal file", pPeer->id);
@@ -412,25 +377,24 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) {
     close(sfd);
     if (code < 0) break;
 
-    index++;
-
-    if (syncAreFilesModified(pPeer) != 0) break;
+    if (syncAreFilesModified(pNode, pPeer) != 0) break;
   }
 
   if (code == 0) {
-    sDebug("%s, wal retrieve is finished", pPeer->id);
     pPeer->sstatus = TAOS_SYNC_STATUS_CACHE;
+    sInfo("%s, wal retrieve is finished, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
+
     SWalHead walHead;
     memset(&walHead, 0, sizeof(walHead));
     code = taosWriteMsg(pPeer->syncFd, &walHead, sizeof(walHead));
   } else {
-    sError("%s, failed to send wal(%s)", pPeer->id, strerror(errno));
+    sError("%s, failed to send wal since %s", pPeer->id, strerror(errno));
   }
 
   return code;
 }
 
-static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) {
+static int32_t syncRetrieveFirstPkt(SSyncPeer *pPeer) {
   SSyncNode *pNode = pPeer->pSyncNode;
 
   SFirstPkt firstPkt;
@@ -440,25 +404,41 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) {
   tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn));
   firstPkt.port = tsSyncPort;
 
-  if (write(pPeer->syncFd, (char *)&firstPkt, sizeof(firstPkt)) < 0) {
-    sError("%s, failed to send syncCmd", pPeer->id);
+  if (taosWriteMsg(pPeer->syncFd, &firstPkt, sizeof(firstPkt)) < 0) {
+    sError("%s, failed to send sync firstPkt since %s", pPeer->id, strerror(errno));
+    return -1;
+  }
+
+  SFirstPktRsp firstPktRsp;
+  if (taosReadMsg(pPeer->syncFd, &firstPktRsp, sizeof(SFirstPktRsp)) < 0) {
+    sError("%s, failed to read sync firstPkt rsp since %s", pPeer->id, strerror(errno));
+    return -1;
+  }
+
+  return 0;
+}
+
+static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) {
+  sInfo("%s, start to retrieve, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
+  if (syncRetrieveFirstPkt(pPeer) < 0) {
+    sError("%s, failed to start retrieve", pPeer->id);
     return -1;
   }
 
   pPeer->sversion = 0;
   pPeer->sstatus = TAOS_SYNC_STATUS_FILE;
-  sDebug("%s, start to retrieve file", pPeer->id);
+  sInfo("%s, start to retrieve files, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
   if (syncRetrieveFile(pPeer) < 0) {
-    sError("%s, failed to retrieve file", pPeer->id);
+    sError("%s, failed to retrieve files", pPeer->id);
     return -1;
   }
 
   // if no files are synced, there must be wal to sync, sversion must be larger than one
   if (pPeer->sversion == 0) pPeer->sversion = 1;
 
-  sDebug("%s, start to retrieve wal", pPeer->id);
+  sInfo("%s, start to retrieve wals", pPeer->id);
   if (syncRetrieveWal(pPeer) < 0) {
-    sError("%s, failed to retrieve wal", pPeer->id);
+    sError("%s, failed to retrieve wals", pPeer->id);
     return -1;
   }
 
@@ -470,7 +450,8 @@ void *syncRetrieveData(void *param) {
   SSyncNode *pNode = pPeer->pSyncNode;
   taosBlockSIGPIPE();
 
-  pPeer->fileChanged = 0;
+  if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, pPeer->numOfRetrieves);
+
   pPeer->syncFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0);
   if (pPeer->syncFd < 0) {
     sError("%s, failed to open socket to sync", pPeer->id);
@@ -478,7 +459,7 @@ void *syncRetrieveData(void *param) {
     sInfo("%s, sync tcp is setup", pPeer->id);
 
     if (syncRetrieveDataStepByStep(pPeer) == 0) {
-      sDebug("%s, sync retrieve process is successful", pPeer->id);
+      sInfo("%s, sync retrieve process is successful", pPeer->id);
     } else {
       sError("%s, failed to retrieve data, restart connection", pPeer->id);
       syncRestartConnection(pPeer);
@@ -486,13 +467,10 @@ void *syncRetrieveData(void *param) {
   }
 
   if (pPeer->fileChanged) {
-    // if file is changed 3 times continuously, start flow control
     pPeer->numOfRetrieves++;
-    if (pPeer->numOfRetrieves >= 2 && pNode->notifyFlowCtrl)
-      (*pNode->notifyFlowCtrl)(pNode->ahandle, 4 << (pPeer->numOfRetrieves - 2));
   } else {
     pPeer->numOfRetrieves = 0;
-    if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->ahandle, 0);
+    if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, 0);
   }
 
   pPeer->fileChanged = 0;

src/sync/src/taosTcpPool.c

@@ -150,7 +150,7 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int32_t connFd) {
 }
 
 void taosFreeTcpConn(void *param) {
-  SConnObj *  pConn = (SConnObj *)param;
+  SConnObj *  pConn = param;
   SThreadObj *pThread = pConn->pThread;
 
   sDebug("%p TCP connection will be closed, fd:%d", pThread, pConn->fd);
@@ -203,16 +203,19 @@ static void *taosProcessTcpData(void *param) {
       assert(pConn);
 
       if (events[i].events & EPOLLERR) {
+        sDebug("conn is broken since EPOLLERR");
         taosProcessBrokenLink(pConn);
         continue;
       }
 
       if (events[i].events & EPOLLHUP) {
+        sDebug("conn is broken since EPOLLHUP");
         taosProcessBrokenLink(pConn);
         continue;
       }
 
       if (events[i].events & EPOLLRDHUP) {
+        sDebug("conn is broken since EPOLLRDHUP");
         taosProcessBrokenLink(pConn);
         continue;
       }

src/sync/src/tarbitrator.c

@@ -115,14 +115,14 @@ static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
 
   SFirstPkt firstPkt;
   if (taosReadMsg(connFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) {
-    sError("failed to read peer first pkt from ip:%s(%s)", ipstr, strerror(errno));
+    sError("failed to read peer first pkt from ip:%s since %s", ipstr, strerror(errno));
     taosCloseSocket(connFd);
     return;
   }
 
-  SNodeConn *pNode = (SNodeConn *)calloc(sizeof(SNodeConn), 1);
+  SNodeConn *pNode = calloc(sizeof(SNodeConn), 1);
   if (pNode == NULL) {
-    sError("failed to allocate memory(%s)", strerror(errno));
+    sError("failed to allocate memory since %s", strerror(errno));
     taosCloseSocket(connFd);
     return;
   }
@@ -146,7 +146,7 @@ static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
 static void arbProcessBrokenLink(void *param) {
   SNodeConn *pNode = param;
 
-  sDebug("%s, TCP link is broken(%s), close connection", pNode->id, strerror(errno));
+  sDebug("%s, TCP link is broken since %s, close connection", pNode->id, strerror(errno));
   tfree(pNode);
 }
 

src/sync/test/syncServer.c

@@ -70,7 +70,7 @@ int writeIntoWal(SWalHead *pHead) {
   return 0;
 }
 
-void confirmForward(void *ahandle, void *mhandle, int32_t code) {
+void confirmForward(int32_t vgId, void *mhandle, int32_t code) {
   SRpcMsg * pMsg = (SRpcMsg *)mhandle;
   SWalHead *pHead = (SWalHead *)(((char *)pMsg->pCont) - sizeof(SWalHead));
 
@@ -227,7 +227,7 @@ void processRequestMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
   taosWriteQitem(qhandle, TAOS_QTYPE_RPC, pTemp);
 }
 
-uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) {
+uint32_t getFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) {
   uint32_t    magic;
   struct stat fstat;
   char        aname[280];
@@ -254,7 +254,7 @@ uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex
   return magic;
 }
 
-int getWalInfo(void *ahandle, char *name, int64_t *index) {
+int getWalInfo(int32_t vgId, char *name, int64_t *index) {
   struct stat fstat;
   char        aname[280];
 
@@ -272,7 +272,7 @@ int getWalInfo(void *ahandle, char *name, int64_t *index) {
   return 1;
 }
 
-int writeToCache(void *ahandle, void *data, int type) {
+int writeToCache(int32_t vgId, void *data, int type) {
   SWalHead *pHead = data;
 
   uDebug("pkt from peer is received, ver:%" PRIu64 " len:%d type:%d", pHead->version, pHead->len, type);
@@ -285,9 +285,9 @@ int writeToCache(void *ahandle, void *data, int type) {
   return 0;
 }
 
-void confirmFwd(void *ahandle, int64_t version) { return; }
+void confirmFwd(int32_t vgId, int64_t version) { return; }
 
-void notifyRole(void *ahandle, int8_t r) {
+void notifyRole(int32_t vgId, int8_t r) {
   role = r;
   printf("current role:%s\n", syncRole[role]);
 }
@@ -296,7 +296,6 @@ void initSync() {
   pCfg->replica = 1;
   pCfg->quorum = 1;
   syncInfo.vgId = 1;
-  syncInfo.ahandle = &syncInfo;
   syncInfo.getFileInfo = getFileInfo;
   syncInfo.getWalInfo = getWalInfo;
   syncInfo.writeToCache = writeToCache;

src/tsdb/src/tsdbMemTable.c

@@ -871,10 +871,10 @@ static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter) {
                 listNEles(pRepo->mem->bufBlockList), pBufBlock->offset, pBufBlock->remain);
 
       if (pBufBlock->offset == 0) {  // return the block to buffer pool
-        tsdbLockRepo(pRepo);
+        if (tsdbLockRepo(pRepo) < 0) return;
         SListNode *pNode = tdListPopTail(pRepo->mem->bufBlockList);
         tdListPrependNode(pBufPool->bufBlockList, pNode);
-        tsdbUnlockRepo(pRepo);
+        if (tsdbUnlockRepo(pRepo) < 0) return;
       }
     } else {
       ASSERT(listNEles(pRepo->mem->extraBuffList) > 0);

src/util/inc/hash.h

@@ -31,16 +31,18 @@ extern "C" {
 typedef void (*_hash_free_fn_t)(void *param);
 
 typedef struct SHashNode {
-//  char             *key;
   struct SHashNode *next;
   uint32_t          hashVal;  // the hash value of key
   uint32_t          keyLen;   // length of the key
-//  char             *data;
+  size_t            dataLen;  // length of data
+  int8_t            count;    // reference count
+  int8_t            removed;  // flag to indicate removed
+  char              data[];
 } SHashNode;
 
-#define GET_HASH_NODE_KEY(_n)  ((char*)(_n) + sizeof(SHashNode))
-#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->keyLen)
-
+#define GET_HASH_NODE_KEY(_n)  ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen)
+#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode))
+#define GET_HASH_PNODE(_n) ((char*)(_n) - sizeof(SHashNode));
 typedef enum SHashLockTypeE {
   HASH_NO_LOCK     = 0,
   HASH_ENTRY_LOCK  = 1,
@@ -65,15 +67,6 @@ typedef struct SHashObj {
   SArray         *pMemBlock;    // memory block allocated for SHashEntry
 } SHashObj;
 
-typedef struct SHashMutableIterator {
-  SHashObj  *pHashObj;
-  int32_t    entryIndex;
-  SHashNode *pCur;
-  SHashNode *pNext;           // current node can be deleted for mutable iterator, so keep the next one before return current
-  size_t     numOfChecked;    // already check number of elements in hash table
-  size_t     numOfEntries;    // number of entries while the iterator is created
-} SHashMutableIterator;
-
 /**
  * init the hash table
  *
@@ -142,33 +135,9 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi
  */
 void taosHashCleanup(SHashObj *pHashObj);
 
-/**
- *
- * @param pHashObj
- * @return
- */
-SHashMutableIterator* taosHashCreateIter(SHashObj *pHashObj);
-
-/**
- *
- * @param iter
- * @return
- */
-bool taosHashIterNext(SHashMutableIterator *iter);
-
-/**
- *
- * @param iter
- * @return
- */
-void *taosHashIterGet(SHashMutableIterator *iter);
-
-/**
- *
- * @param iter
- * @return
- */
-void* taosHashDestroyIter(SHashMutableIterator* iter);
+/*
+void *SHashMutableIterator* taosHashCreateIter(SHashObj *pHashObj, void *);
+*/
 
 /**
  *
@@ -179,6 +148,9 @@ int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj);
 
 size_t taosHashGetMemSize(const SHashObj *pHashObj);
 
+void *taosHashIterate(SHashObj *pHashObj, void *p);
+void  taosHashCancelIterate(SHashObj *pHashObj, void *p);
+
 #ifdef __cplusplus
 }
 #endif

src/util/inc/tfile.h

@@ -20,23 +20,26 @@
 extern "C" {
 #endif
 
-#include <unistd.h>
-
 // init taos file module
-int32_t tfinit();
+int32_t tfInit();
 
 // clean up taos file module
-void tfcleanup();
+void tfCleanup();
 
 // the same syntax as UNIX standard open/close/read/write
 // but FD is int64_t and will never be reused
-int64_t tfopen(const char *pathname, int32_t flags);
-int64_t tfclose(int64_t tfd);
-int64_t tfwrite(int64_t tfd, void *buf, int64_t count);
-int64_t tfread(int64_t tfd, void *buf, int64_t count);
+int64_t tfOpen(const char *pathname, int32_t flags);
+int64_t tfOpenM(const char *pathname, int32_t flags, mode_t mode);
+int64_t tfClose(int64_t tfd);
+int64_t tfWrite(int64_t tfd, void *buf, int64_t count);
+int64_t tfRead(int64_t tfd, void *buf, int64_t count);
+int32_t tfFsync(int64_t tfd);
+bool    tfValid(int64_t tfd);
+int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence);
+int32_t tfFtruncate(int64_t tfd, int64_t length);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif  // TDENGINE_TREF_H
+#endif  // TDENGINE_TFILE_H

src/util/src/hash.c

@@ -76,7 +76,7 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) {
 static FORCE_INLINE SHashNode *doSearchInEntryList(SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) {
   SHashNode *pNode = pe->next;
   while (pNode) {
-    if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) {
+    if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) {
       assert(pNode->hashVal == hashVal);
       break;
     }
@@ -114,15 +114,25 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p
  * @param dsize   size of actual data
  * @return        hash node
  */
-static FORCE_INLINE SHashNode *doUpdateHashNode(SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) {
+static FORCE_INLINE SHashNode *doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) {
   assert(pNode->keyLen == pNewNode->keyLen);
+
+  pNode->count--;
   if (prev != NULL) {
     prev->next = pNewNode;
   } else {
     pe->next = pNewNode;
   }
 
+  if (pNode->count <= 0) {
     pNewNode->next = pNode->next;
+    DO_FREE_HASH_NODE(pNode);
+  } else {
+    pNewNode->next = pNode;    
+    pe->num++;
+    atomic_add_fetch_64(&pHashObj->size, 1);
+  }
+  
   return pNewNode;
 }
 
@@ -139,7 +149,6 @@ static void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode);
  * @param pIter
  * @return
  */
-static SHashNode *getNextHashNode(SHashMutableIterator *pIter);
 
 SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) {
   if (capacity == 0 || fn == NULL) {
@@ -213,7 +222,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da
 
   SHashNode* prev = NULL;
   while (pNode) {
-    if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) {
+    if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) {
       assert(pNode->hashVal == hashVal);
       break;
     }
@@ -244,8 +253,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da
   } else {
     // not support the update operation, return error
     if (pHashObj->enableUpdate) {
-      doUpdateHashNode(pe, prev, pNode, pNewNode);
-      DO_FREE_HASH_NODE(pNode);
+      doUpdateHashNode(pHashObj, pe, prev, pNode, pNewNode);
     } else {
       DO_FREE_HASH_NODE(pNewNode);
     }
@@ -335,22 +343,10 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe
   int32_t     slot = HASH_INDEX(hashVal, pHashObj->capacity);
   SHashEntry *pe = pHashObj->hashList[slot];
 
-  // no data, return directly
-  if (pe->num == 0) {
-    __rd_unlock(&pHashObj->lock, pHashObj->type);
-    return -1;
-  }
-
   if (pHashObj->type == HASH_ENTRY_LOCK) {
     taosWLockLatch(&pe->latch);
   }
 
-  if (pe->num == 0) {
-    assert(pe->next == NULL);
-  } else {
-    assert(pe->next != NULL);
-  }
-
   // double check after locked
   if (pe->num == 0) {
     assert(pe->next == NULL);
@@ -360,36 +356,36 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe
     return -1;
   }
 
+  int code = -1;
   SHashNode *pNode = pe->next;
-  SHashNode *pRes = NULL;
+  SHashNode *prevNode = NULL;
 
-  // remove it
-  if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) {
-    pe->num -= 1;
-    pRes = pNode;
-    pe->next = pNode->next;
-  } else {
-    while (pNode->next != NULL) {
-      if (((pNode->next)->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY((pNode->next)), key, keyLen) == 0)) {
-        assert((pNode->next)->hashVal == hashVal);
+  while (pNode) {
+    if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) 
       break;
-      }
 
+    prevNode = pNode;
     pNode = pNode->next;
   }
 
+  if (pNode) {
+    code = 0;  // it is found
 
-    if (pNode->next != NULL) {
-      pe->num -= 1;
-      pRes = pNode->next;
-      pNode->next = pNode->next->next;
-    }
+    pNode->count--;
+    pNode->removed = 1;
+    if (pNode->count <= 0) {
+      if (prevNode) {
+        prevNode->next = pNode->next;
+      } else {
+        pe->next = pNode->next;
       }
    
-  if (pe->num == 0) {
-    assert(pe->next == NULL);
-  } else {
-    assert(pe->next != NULL);
+      if (data) memcpy(data, GET_HASH_NODE_DATA(pNode), dsize);
+
+      pe->num--;
+      atomic_sub_fetch_64(&pHashObj->size, 1);
+      FREE_HASH_NODE(pHashObj, pNode);
+    }
   } 
 
   if (pHashObj->type == HASH_ENTRY_LOCK) {
@@ -398,17 +394,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe
 
   __rd_unlock(&pHashObj->lock, pHashObj->type);
 
-  if (data != NULL && pRes != NULL) {
-    memcpy(data, GET_HASH_NODE_DATA(pRes), dsize);
-  }
-
-  if (pRes != NULL) {
-    atomic_sub_fetch_64(&pHashObj->size, 1);
-    FREE_HASH_NODE(pHashObj, pRes);
-    return 0;
-  } else {
-    return -1;
-  }
+  return code;
 }
 
 int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param) {
@@ -531,98 +517,6 @@ void taosHashCleanup(SHashObj *pHashObj) {
   free(pHashObj);
 }
 
-SHashMutableIterator *taosHashCreateIter(SHashObj *pHashObj) {
-  SHashMutableIterator *pIter = calloc(1, sizeof(SHashMutableIterator));
-  if (pIter == NULL) {
-    return NULL;
-  }
-
-  pIter->pHashObj = pHashObj;
-
-  // keep it in local variable, in case the resize operation expand the size
-  pIter->numOfEntries = pHashObj->capacity;
-  return pIter;
-}
-
-bool taosHashIterNext(SHashMutableIterator *pIter) {
-  if (pIter == NULL) {
-    return false;
-  }
-
-  size_t size = taosHashGetSize(pIter->pHashObj);
-  if (size == 0) {
-    return false;
-  }
-
-  // check the first one
-  if (pIter->numOfChecked == 0) {
-    assert(pIter->pCur == NULL && pIter->pNext == NULL);
-
-    while (1) {
-      SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex];
-      if (pEntry->num == 0) {
-        assert(pEntry->next == NULL);
-
-        pIter->entryIndex++;
-        continue;
-      }
-
-      if (pIter->pHashObj->type == HASH_ENTRY_LOCK) {
-        taosRLockLatch(&pEntry->latch);
-      }
-
-      pIter->pCur = pEntry->next;
-
-      if (pIter->pCur->next) {
-        pIter->pNext = pIter->pCur->next;
-
-        if (pIter->pHashObj->type == HASH_ENTRY_LOCK) {
-          taosRUnLockLatch(&pEntry->latch);
-        }
-      } else {
-        if (pIter->pHashObj->type == HASH_ENTRY_LOCK) {
-          taosRUnLockLatch(&pEntry->latch);
-        }
-
-        pIter->pNext = getNextHashNode(pIter);
-      }
-
-      break;
-    }
-
-    pIter->numOfChecked++;
-    return true;
-  } else {
-    assert(pIter->pCur != NULL);
-    if (pIter->pNext) {
-      pIter->pCur = pIter->pNext;
-    } else {  // no more data in the hash list
-      return false;
-    }
-
-    pIter->numOfChecked++;
-
-    if (pIter->pCur->next) {
-      pIter->pNext = pIter->pCur->next;
-    } else {
-      pIter->pNext = getNextHashNode(pIter);
-    }
-
-    return true;
-  }
-}
-
-void *taosHashIterGet(SHashMutableIterator *iter) { return (iter == NULL) ? NULL : GET_HASH_NODE_DATA(iter->pCur); }
-
-void *taosHashDestroyIter(SHashMutableIterator *iter) {
-  if (iter == NULL) {
-    return NULL;
-  }
-
-  free(iter);
-  return NULL;
-}
-
 // for profile only
 int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) {
   if (pHashObj == NULL || pHashObj->size == 0) {
@@ -759,6 +653,8 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s
 
   pNewNode->keyLen = (uint32_t)keyLen;
   pNewNode->hashVal = hashVal;
+  pNewNode->dataLen = dsize;
+  pNewNode->count = 1;
 
   memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize);
   memcpy(GET_HASH_NODE_KEY(pNewNode), key, keyLen);
@@ -775,39 +671,136 @@ void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode) {
   pEntry->num += 1;
 }
 
-SHashNode *getNextHashNode(SHashMutableIterator *pIter) {
-  assert(pIter != NULL);
+size_t taosHashGetMemSize(const SHashObj *pHashObj) {
+  if (pHashObj == NULL) {
+    return 0;
+  }
+
+  return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj);
+}
 
-  pIter->entryIndex++;
-  SHashNode *p = NULL;
+// release the pNode, return next pNode, and lock the current entry
+static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) {
 
-  while (pIter->entryIndex < pIter->numOfEntries) {
-    SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex];
-    if (pEntry->num == 0) {
-      pIter->entryIndex++;
-      continue;
+  SHashNode *pOld = (SHashNode *)GET_HASH_PNODE(p);
+  SHashNode *prevNode = NULL;
+
+  *slot = HASH_INDEX(pOld->hashVal, pHashObj->capacity);
+  SHashEntry *pe = pHashObj->hashList[*slot];
+
+  // lock entry
+  if (pHashObj->type == HASH_ENTRY_LOCK) {
+    taosWLockLatch(&pe->latch);
   }
 
-    if (pIter->pHashObj->type == HASH_ENTRY_LOCK) {
-      taosRLockLatch(&pEntry->latch);
+  SHashNode *pNode = pe->next;
+
+  while (pNode) {
+    if (pNode == pOld) 
+      break;
+
+    prevNode = pNode;
+    pNode = pNode->next;
   }
 
-    p = pEntry->next;
+  if (pNode) { 
+    pNode = pNode->next;
+    while (pNode) {
+      if (pNode->removed == 0) break;
+      pNode = pNode->next;
+    }
 
-    if (pIter->pHashObj->type == HASH_ENTRY_LOCK) {
-      taosRUnLockLatch(&pEntry->latch);
+    pOld->count--;
+    if (pOld->count <=0) {
+      if (prevNode) {
+        prevNode->next = pOld->next;
+      } else {
+        pe->next = pOld->next;
       }
    
-    return p;
+      pe->num--;
+      atomic_sub_fetch_64(&pHashObj->size, 1);
+      FREE_HASH_NODE(pHashObj, pOld);
+    } 
+  } else {
+    uError("pNode:%p data:%p is not there!!!", pNode, p);
   }
 
-  return NULL;
+  return pNode;
 }
 
-size_t taosHashGetMemSize(const SHashObj *pHashObj) {
-  if (pHashObj == NULL) {
-    return 0;
+void *taosHashIterate(SHashObj *pHashObj, void *p) {
+  if (pHashObj == NULL) return NULL; 
+
+  int  slot = 0;
+  char *data = NULL;
+
+  // only add the read lock to disable the resize process
+  __rd_lock(&pHashObj->lock, pHashObj->type);
+
+  SHashNode *pNode = NULL;
+  if (p) {
+    pNode = taosHashReleaseNode(pHashObj, p, &slot);
+    if (pNode == NULL) {
+      SHashEntry *pe = pHashObj->hashList[slot];
+      if (pHashObj->type == HASH_ENTRY_LOCK) {
+        taosWUnLockLatch(&pe->latch);
       } 
 
-  return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj);
+      slot = slot + 1;
+    }
+  }
+
+  if (pNode == NULL) {
+    for (; slot < pHashObj->capacity; ++slot) {
+      SHashEntry *pe = pHashObj->hashList[slot];
+
+      // lock entry
+      if (pHashObj->type == HASH_ENTRY_LOCK) {
+        taosWLockLatch(&pe->latch);
+      }
+
+      pNode = pe->next;
+      while (pNode) {
+        if (pNode->removed == 0) break;
+        pNode = pNode->next;
+      }
+
+      if (pNode) break;
+
+      if (pHashObj->type == HASH_ENTRY_LOCK) {
+        taosWUnLockLatch(&pe->latch);
+      } 
+    }
+  }
+
+  if (pNode) {
+    SHashEntry *pe = pHashObj->hashList[slot];
+    pNode->count++;
+    data = GET_HASH_NODE_DATA(pNode);
+    if (pHashObj->type == HASH_ENTRY_LOCK) {
+      taosWUnLockLatch(&pe->latch);
+    } 
+  }
+
+  __rd_unlock(&pHashObj->lock, pHashObj->type);
+  return data;
+
+}
+
+void taosHashCancelIterate(SHashObj *pHashObj, void *p) {
+  if (pHashObj == NULL || p == NULL) return;
+
+  // only add the read lock to disable the resize process
+  __rd_lock(&pHashObj->lock, pHashObj->type);
+
+  int slot;
+  taosHashReleaseNode(pHashObj, p, &slot);
+
+  SHashEntry *pe = pHashObj->hashList[slot];
+  if (pHashObj->type == HASH_ENTRY_LOCK) {
+    taosWUnLockLatch(&pe->latch);
+  } 
+
+  __rd_unlock(&pHashObj->lock, pHashObj->type);
 }

src/util/src/tfile.c

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define _DEFAULT_SOURCE
 #include "os.h"
 #include "taoserror.h"
 #include "tulog.h"
@@ -21,40 +22,52 @@
 
 static int32_t tsFileRsetId = -1;
 
-static void taosCloseFile(void *p) {
+static void tfCloseFile(void *p) {
   close((int32_t)(uintptr_t)p);
 }
 
-int32_t tfinit() {
-  tsFileRsetId = taosOpenRef(2000, taosCloseFile);
-  return tsFileRsetId;
+int32_t tfInit() {
+  tsFileRsetId = taosOpenRef(2000, tfCloseFile);
+  if (tsFileRsetId > 0) {
+    return 0;
+  } else {
+    return -1;
+  }
 }
 
-void tfcleanup() {
+void tfCleanup() {
   if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId);
   tsFileRsetId = -1;
 }
 
-int64_t tfopen(const char *pathname, int32_t flags) {
-  int32_t fd = open(pathname, flags);
-
+static int64_t tfOpenImp(int32_t fd) {
   if (fd < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
 
-  void *p = (void *)(int64_t)fd;
+  void *  p = (void *)(int64_t)fd;
   int64_t rid = taosAddRef(tsFileRsetId, p);
   if (rid < 0) close(fd);
 
   return rid;
 }
 
-int64_t tfclose(int64_t tfd) {
+int64_t tfOpen(const char *pathname, int32_t flags) {
+  int32_t fd = open(pathname, flags);
+  return tfOpenImp(fd);
+}
+
+int64_t tfOpenM(const char *pathname, int32_t flags, mode_t mode) {
+  int32_t fd = open(pathname, flags, mode);
+  return tfOpenImp(fd);
+}
+
+int64_t tfClose(int64_t tfd) {
   return taosRemoveRef(tsFileRsetId, tfd);
 }
 
-int64_t tfwrite(int64_t tfd, void *buf, int64_t count) {
+int64_t tfWrite(int64_t tfd, void *buf, int64_t count) {
   void *p = taosAcquireRef(tsFileRsetId, tfd);
   if (p == NULL) return -1;
 
@@ -67,7 +80,7 @@ int64_t tfwrite(int64_t tfd, void *buf, int64_t count) {
   return ret;
 }
 
-int64_t tfread(int64_t tfd, void *buf, int64_t count) {
+int64_t tfRead(int64_t tfd, void *buf, int64_t count) {
   void *p = taosAcquireRef(tsFileRsetId, tfd);
   if (p == NULL) return -1;
 
@@ -79,3 +92,32 @@ int64_t tfread(int64_t tfd, void *buf, int64_t count) {
   taosReleaseRef(tsFileRsetId, tfd);
   return ret;
 }
+
+int64_t tfFsync(int64_t tfd) {
+  void *p = taosAcquireRef(tsFileRsetId, tfd);
+  if (p == NULL) return -1;
+
+  int32_t fd = (int32_t)(uintptr_t)p;
+  return fsync(fd);
+}
+
+bool tfValid(int64_t tfd) {
+  void *p = taosAcquireRef(tsFileRsetId, tfd);
+  return p != NULL;
+}
+
+int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) {
+  void *p = taosAcquireRef(tsFileRsetId, tfd);
+  if (p == NULL) return -1;
+
+  int32_t fd = (int32_t)(uintptr_t)p;
+  return taosLSeek(fd, offset, whence);
+}
+
+int32_t tfFtruncate(int64_t tfd, int64_t length) {
+  void *p = taosAcquireRef(tsFileRsetId, tfd);
+  if (p == NULL) return -1;
+
+  int32_t fd = (int32_t)(uintptr_t)p;
+  return taosFtruncate(fd, length);
+}

src/util/src/tkvstore.c

@@ -529,7 +529,7 @@ static int tdRestoreKVStore(SKVStore *pStore) {
   void *                buf = NULL;
   int64_t               maxBufSize = 0;
   SKVRecord             rInfo = {0};
-  SHashMutableIterator *pIter = NULL;
+  SKVRecord            *pRecord = NULL;
 
   ASSERT(TD_KVSTORE_HEADER_SIZE == lseek(pStore->fd, 0, SEEK_CUR));
   ASSERT(pStore->info.size == TD_KVSTORE_HEADER_SIZE);
@@ -582,16 +582,8 @@ static int tdRestoreKVStore(SKVStore *pStore) {
     goto _err;
   }
 
-  pIter = taosHashCreateIter(pStore->map);
-  if (pIter == NULL) {
-    uError("failed to create hash iter while opening KV store %s", pStore->fname);
-    terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
-    goto _err;
-  }
-
-  while (taosHashIterNext(pIter)) {
-    SKVRecord *pRecord = taosHashIterGet(pIter);
-
+  pRecord = taosHashIterate(pStore->map, NULL);
+  while (pRecord) {
     if (lseek(pStore->fd, (off_t)(pRecord->offset + sizeof(SKVRecord)), SEEK_SET) < 0) {
       uError("failed to lseek file %s since %s, offset %" PRId64, pStore->fname, strerror(errno), pRecord->offset);
       terrno = TAOS_SYSTEM_ERROR(errno);
@@ -613,16 +605,17 @@ static int tdRestoreKVStore(SKVStore *pStore) {
         goto _err;
       }
     }
+
+    pRecord = taosHashIterate(pStore->map, pRecord);
   }
 
   if (pStore->aFunc) (*pStore->aFunc)(pStore->appH);
 
-  taosHashDestroyIter(pIter);
   tfree(buf);
   return 0;
 
 _err:
-  taosHashDestroyIter(pIter);
+  taosHashCancelIterate(pStore->map, pRecord);
   tfree(buf);
   return -1;
 }

src/util/src/tqueue.c

@@ -79,15 +79,12 @@ void taosCloseQueue(taos_queue param) {
 
   if (queue->qset) taosRemoveFromQset(qset, queue); 
 
-  pthread_mutex_lock(&queue->mutex);
-
   while (pNode) {
     pTemp = pNode;
     pNode = pNode->next;
     free (pTemp);
   }
 
-  pthread_mutex_unlock(&queue->mutex);
   pthread_mutex_destroy(&queue->mutex);
   free(queue);
 

src/util/src/tskiplist.c

@@ -291,7 +291,10 @@ bool tSkipListIterNext(SSkipListIterator *iter) {
     iter->next = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0);
     iter->step++;
   } else {
-    if (iter->cur == pSkipList->pHead) return false;
+    if (iter->cur == pSkipList->pHead) {
+      tSkipListUnlock(pSkipList);
+      return false;
+    }
     iter->cur = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0);
 
     // a new node is inserted into between iter->cur and iter->next, ignore it

src/util/src/tsocket.c

@@ -107,7 +107,7 @@ int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) {
   while (nleft > 0) {
     nwritten = (int32_t)taosWriteSocket(fd, (char *)ptr, (size_t)nleft);
     if (nwritten <= 0) {
-      if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK)
+      if (errno == EINTR /* || errno == EAGAIN || errno == EWOULDBLOCK */)
         continue;
       else
         return -1;
@@ -133,7 +133,7 @@ int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) {
     if (nread == 0) {
       break;
     } else if (nread < 0) {
-      if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
+      if (errno == EINTR/* || errno == EAGAIN || errno == EWOULDBLOCK*/) {
         continue;
       } else {
         return -1;

src/vnode/inc/vnodeInt.h

@@ -39,11 +39,12 @@ typedef struct {
   int32_t      refCount;  // reference count
   int32_t      queuedWMsg;
   int32_t      queuedRMsg;
-  int32_t      delayMs;
+  int32_t      flowctrlLevel;
   int8_t       status; 
   int8_t       role;   
   int8_t       accessState;
   int8_t       isFull;
+  int8_t       isCommiting;
   uint64_t     version;   // current version 
   uint64_t     fversion;  // version on saved data file
   void        *wqueue;

src/vnode/src/vnodeMain.c

@@ -30,19 +30,22 @@
 
 static SHashObj*tsVnodesHash;
 static void     vnodeCleanUp(SVnodeObj *pVnode);
-static int      vnodeProcessTsdbStatus(void *arg, int status, int eno);
-static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion);
-static int      vnodeGetWalInfo(void *ahandle, char *fileName, int64_t *fileId);
-static void     vnodeNotifyRole(void *ahandle, int8_t role);
-static void     vnodeCtrlFlow(void *handle, int32_t mseconds); 
-static int      vnodeNotifyFileSynced(void *ahandle, uint64_t fversion);
+static int32_t  vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno);
+static uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion);
+static int32_t  vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId);
+static void     vnodeNotifyRole(int32_t vgId, int8_t role);
+static void     vnodeCtrlFlow(int32_t vgId, int32_t level);
+static int32_t  vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion);
+static void     vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code);
+static int32_t  vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam);
+static int32_t  vnodeGetFileVersion(int32_t vgId, uint64_t *fver);
 
 #ifndef _SYNC
 int64_t syncStart(const SSyncInfo *info) { return NULL; }
-int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int qtype) { return 0; }
+int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype) { return 0; }
 void    syncStop(int64_t rid) {}
-int32_t syncReconfig(int64_t rid, const SSyncCfg * cfg) { return 0; }
-int     syncGetNodesRole(int64_t rid, SNodesRole * cfg) { return 0; }
+int32_t syncReconfig(int64_t rid, const SSyncCfg *cfg) { return 0; }
+int32_t syncGetNodesRole(int64_t rid, SNodesRole *cfg) { return 0; }
 void    syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {}
 #endif
 
@@ -55,13 +58,13 @@ char* vnodeStatus[] = {
 };
 
 int32_t vnodeInitResources() {
-  int code = syncInit();
+  int32_t code = syncInit();
   if (code != 0) return code;
 
   vnodeInitWriteFp();
   vnodeInitReadFp();
 
-  tsVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true);
+  tsVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
   if (tsVnodesHash == NULL) {
     vError("failed to init vnode list");
     return TSDB_CODE_VND_OUT_OF_MEMORY;
@@ -173,8 +176,8 @@ int32_t vnodeDrop(int32_t vgId) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t vnodeAlter(void *param, SCreateVnodeMsg *pVnodeCfg) {
-  SVnodeObj *pVnode = param;
+int32_t vnodeAlter(void *vparam, SCreateVnodeMsg *pVnodeCfg) {
+  SVnodeObj *pVnode = vparam;
 
   // vnode in non-ready state and still needs to return success instead of TSDB_CODE_VND_INVALID_STATUS
   // cfgVersion can be corrected by status msg
@@ -325,19 +328,32 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
   walRemoveAllOldFiles(pVnode->wal);
   walRenew(pVnode->wal);
 
+  pVnode->qMgmt = qOpenQueryMgmt(pVnode->vgId);
+  if (pVnode->qMgmt == NULL) {
+    vnodeCleanUp(pVnode);
+    return terrno;
+  }
+
+  pVnode->events = NULL;
+
+  vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode);
+  tsdbIncCommitRef(pVnode->vgId);
+
+  taosHashPut(tsVnodesHash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *));
+
   SSyncInfo syncInfo;
   syncInfo.vgId = pVnode->vgId;
   syncInfo.version = pVnode->version;
   syncInfo.syncCfg = pVnode->syncCfg;
   sprintf(syncInfo.path, "%s", rootDir);
-  syncInfo.ahandle = pVnode;
   syncInfo.getWalInfo = vnodeGetWalInfo;
   syncInfo.getFileInfo = vnodeGetFileInfo;
-  syncInfo.writeToCache = vnodeWriteToWQueue;
-  syncInfo.confirmForward = dnodeSendRpcVWriteRsp; 
+  syncInfo.writeToCache = vnodeWriteToCache;
+  syncInfo.confirmForward = vnodeConfirmForard; 
   syncInfo.notifyRole = vnodeNotifyRole;
   syncInfo.notifyFlowCtrl = vnodeCtrlFlow;
   syncInfo.notifyFileSynced = vnodeNotifyFileSynced;
+  syncInfo.getFileVersion = vnodeGetFileVersion;
   pVnode->sync = syncStart(&syncInfo);
 
 #ifndef _SYNC
@@ -346,24 +362,13 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
   if (pVnode->sync <= 0) {
     vError("vgId:%d, failed to open sync module, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica,
            tstrerror(terrno));
+    vnodeRelease(pVnode);
     vnodeCleanUp(pVnode);
     return terrno;
   }
 #endif
 
-  pVnode->qMgmt = qOpenQueryMgmt(pVnode->vgId);
-  if (pVnode->qMgmt == NULL) {
-    vnodeCleanUp(pVnode);
-    return terrno;
-  }
-
-  pVnode->events = NULL;
   pVnode->status = TAOS_VN_STATUS_READY;
-  vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode);
-
-  tsdbIncCommitRef(pVnode->vgId);
-  taosHashPut(tsVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *));
-
   return TSDB_CODE_SUCCESS;
 }
 
@@ -389,7 +394,7 @@ void vnodeRelease(void *vparam) {
   assert(refCount >= 0);
 
   if (refCount > 0) {
-    if (pVnode->status == TAOS_VN_STATUS_RESET && refCount == 2) {
+    if (pVnode->status == TAOS_VN_STATUS_RESET && refCount <= 3) {
       tsem_post(&pVnode->sem);
     }
     return;
@@ -517,11 +522,10 @@ static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) {
 }
 
 int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
-  SHashMutableIterator *pIter = taosHashCreateIter(tsVnodesHash);
-  while (taosHashIterNext(pIter)) {
-    SVnodeObj **pVnode = taosHashIterGet(pIter);
-    if (pVnode == NULL) continue;
-    if (*pVnode == NULL) continue;
+  void *pIter = taosHashIterate(tsVnodesHash, NULL);
+  while (pIter) {
+    SVnodeObj **pVnode = pIter;
+    if (*pVnode) {
 
     (*numOfVnodes)++;
     if (*numOfVnodes >= TSDB_MAX_VNODES) {
@@ -530,25 +534,25 @@ int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
     } else {
       vnodeList[*numOfVnodes - 1] = (*pVnode)->vgId;
     }
+
     }
 
-  taosHashDestroyIter(pIter);
+    pIter = taosHashIterate(tsVnodesHash, pIter);    
+  }
   return TSDB_CODE_SUCCESS;
 }
 
 void vnodeBuildStatusMsg(void *param) {
   SStatusMsg *pStatus = param;
-  SHashMutableIterator *pIter = taosHashCreateIter(tsVnodesHash);
-
-  while (taosHashIterNext(pIter)) {
-    SVnodeObj **pVnode = taosHashIterGet(pIter);
-    if (pVnode == NULL) continue;
-    if (*pVnode == NULL) continue;
 
+  void *pIter = taosHashIterate(tsVnodesHash, NULL);
+  while (pIter) {
+    SVnodeObj **pVnode = pIter;
+    if (*pVnode) {
       vnodeBuildVloadMsg(*pVnode, pStatus);
     }
-
-  taosHashDestroyIter(pIter);
+    pIter = taosHashIterate(tsVnodesHash, pIter);
+  }
 }
 
 void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes) {
@@ -567,11 +571,11 @@ void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes) {
 
 static void vnodeCleanUp(SVnodeObj *pVnode) {
   // remove from hash, so new messages wont be consumed
-  taosHashRemove(tsVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t));
+  taosHashRemove(tsVnodesHash, &pVnode->vgId, sizeof(int32_t));
 
   if (pVnode->status != TAOS_VN_STATUS_INIT) {
     // it may be in updateing or reset state, then it shall wait
-    int i = 0;
+    int32_t i = 0;
     while (atomic_val_compare_exchange_8(&pVnode->status, TAOS_VN_STATUS_READY, TAOS_VN_STATUS_CLOSING) !=
            TAOS_VN_STATUS_READY) {
       if (++i % 1000 == 0) {
@@ -594,51 +598,75 @@ static void vnodeCleanUp(SVnodeObj *pVnode) {
   vnodeRelease(pVnode);
 }
 
-// TODO: this is a simple implement
-static int vnodeProcessTsdbStatus(void *arg, int status, int eno) {
+static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) {
   SVnodeObj *pVnode = arg;
 
   if (eno != TSDB_CODE_SUCCESS) {
     vError("vgId:%d, failed to commit since %s, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, tstrerror(eno),
            pVnode->fversion, pVnode->version);
+    pVnode->isCommiting = 0;
     pVnode->isFull = 1;
     return 0;
   }
 
   if (status == TSDB_STATUS_COMMIT_START) {
+    pVnode->isCommiting = 1;
     pVnode->fversion = pVnode->version;
     vDebug("vgId:%d, start commit, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version);
-    if (pVnode->status == TAOS_VN_STATUS_INIT) {
-      return 0;
-    } else {
+    if (pVnode->status != TAOS_VN_STATUS_INIT) {
       return walRenew(pVnode->wal);
     }
+    return 0;
   }
 
   if (status == TSDB_STATUS_COMMIT_OVER) {
     vDebug("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version);
+    pVnode->isCommiting = 0;
     pVnode->isFull = 0;
+    if (pVnode->status != TAOS_VN_STATUS_INIT) {
       walRemoveOneOldFile(pVnode->wal);
+    }
     return vnodeSaveVersion(pVnode);
   }
 
   return 0;
 }
 
-static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size,
+static uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size,
                                  uint64_t *fversion) {
-  SVnodeObj *pVnode = ahandle;
+  SVnodeObj *pVnode = vnodeAcquire(vgId);
+  if (pVnode == NULL) {
+    vError("vgId:%d, vnode not found while get file info", vgId);
+    return 0;
+  }
+
   *fversion = pVnode->fversion;
-  return tsdbGetFileInfo(pVnode->tsdb, name, index, eindex, size);
+  uint32_t ret = tsdbGetFileInfo(pVnode->tsdb, name, index, eindex, size);
+
+  vnodeRelease(pVnode);
+  return ret;
 }
 
-static int vnodeGetWalInfo(void *ahandle, char *fileName, int64_t *fileId) {
-  SVnodeObj *pVnode = ahandle;
-  return walGetWalFile(pVnode->wal, fileName, fileId);
+static int32_t vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId) {
+  SVnodeObj *pVnode = vnodeAcquire(vgId);
+  if (pVnode == NULL) {
+    vError("vgId:%d, vnode not found while get wal info", vgId);
+    return -1;
+  }
+
+  int32_t code = walGetWalFile(pVnode->wal, fileName, fileId);
+
+  vnodeRelease(pVnode);
+  return code;
 }
 
-static void vnodeNotifyRole(void *ahandle, int8_t role) {
-  SVnodeObj *pVnode = ahandle;
+static void vnodeNotifyRole(int32_t vgId, int8_t role) {
+  SVnodeObj *pVnode = vnodeAcquire(vgId);
+  if (pVnode == NULL) {
+    vTrace("vgId:%d, vnode not found while notify role", vgId);
+    return;
+  }
+
   vInfo("vgId:%d, sync role changed from %s to %s", pVnode->vgId, syncRole[pVnode->role], syncRole[role]);
   pVnode->role = role;
   dnodeSendStatusMsgToMnode();
@@ -648,17 +676,26 @@ static void vnodeNotifyRole(void *ahandle, int8_t role) {
   } else {
     cqStop(pVnode->cq);
   }
+
+  vnodeRelease(pVnode);
 }
 
-static void vnodeCtrlFlow(void *ahandle, int32_t mseconds) {
-  SVnodeObj *pVnode = ahandle;
-  if (pVnode->delayMs != mseconds) {
-    pVnode->delayMs = mseconds;
-    vDebug("vgId:%d, sync flow control, mseconds:%d", pVnode->vgId, mseconds);
+static void vnodeCtrlFlow(int32_t vgId, int32_t level) {
+  SVnodeObj *pVnode = vnodeAcquire(vgId);
+  if (pVnode == NULL) {
+    vTrace("vgId:%d, vnode not found while flow ctrl", vgId);
+    return;
   }
+
+  if (pVnode->flowctrlLevel != level) {
+    vDebug("vgId:%d, set flowctrl level from %d to %d", pVnode->vgId, pVnode->flowctrlLevel, level);
+    pVnode->flowctrlLevel = level;
+  }
+
+  vnodeRelease(pVnode);
 }
 
-static int vnodeResetTsdb(SVnodeObj *pVnode) {
+static int32_t vnodeResetTsdb(SVnodeObj *pVnode) {
   char rootDir[128] = "\0";
   sprintf(rootDir, "%s/tsdb", pVnode->rootDir);
 
@@ -672,7 +709,7 @@ static int vnodeResetTsdb(SVnodeObj *pVnode) {
   // acquire vnode
   int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
 
-  if (refCount > 2) {
+  if (refCount > 3) {
     tsem_wait(&pVnode->sem);
   }
 
@@ -692,14 +729,63 @@ static int vnodeResetTsdb(SVnodeObj *pVnode) {
   return 0;
 }
 
-static int vnodeNotifyFileSynced(void *ahandle, uint64_t fversion) {
-  SVnodeObj *pVnode = ahandle;
+static int32_t vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion) {
+  SVnodeObj *pVnode = vnodeAcquire(vgId);
+  if (pVnode == NULL) {
+    vError("vgId:%d, vnode not found while notify file synced", vgId);
+    return 0;
+  }
 
   pVnode->fversion = fversion;
   pVnode->version = fversion;
   vnodeSaveVersion(pVnode);
 
-  vDebug("vgId:%d, data file is synced, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion,
-         pVnode->version);
-  return vnodeResetTsdb(pVnode);
+  vDebug("vgId:%d, data file is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion);
+  int32_t code = vnodeResetTsdb(pVnode);
+
+  vnodeRelease(pVnode);
+  return code;
+}
+
+void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code) {
+  void *pVnode = vnodeAcquire(vgId);
+  if (pVnode == NULL) {
+    vError("vgId:%d, vnode not found while confirm forward", vgId);
+    return;
+  }
+
+  dnodeSendRpcVWriteRsp(pVnode, wparam, code);
+  vnodeRelease(pVnode);
+}
+
+static int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam) {
+  SVnodeObj *pVnode = vnodeAcquire(vgId);
+  if (pVnode == NULL) {
+    vError("vgId:%d, vnode not found while write to cache", vgId);
+    return TSDB_CODE_VND_INVALID_VGROUP_ID;
+  }
+
+  int32_t code = vnodeWriteToWQueue(pVnode, wparam, qtype, rparam);
+
+  vnodeRelease(pVnode);
+  return code;
+}
+
+static int32_t vnodeGetFileVersion(int32_t vgId, uint64_t *fver) {
+  SVnodeObj *pVnode = vnodeAcquire(vgId);
+  if (pVnode == NULL) {
+    vError("vgId:%d, vnode not found while write to cache", vgId);
+    return -1;
+  }
+
+  int32_t code = 0;
+  if (pVnode->isCommiting) {
+    vDebug("vgId:%d, vnode is commiting while get file version", vgId);
+    code = -1;
+  } else {
+    *fver = pVnode->fversion;
+  }
+
+  vnodeRelease(pVnode);
+  return code;
 }

src/vnode/src/vnodeRead.c

@@ -53,8 +53,7 @@ int32_t vnodeProcessRead(void *vparam, SVReadMsg *pRead) {
   return (*vnodeProcessReadMsgFp[msgType])(pVnode, pRead);
 }
 
-static int32_t vnodeCheckRead(void *vparam) {
-  SVnodeObj *pVnode = vparam;
+static int32_t vnodeCheckRead(SVnodeObj *pVnode) {
   if (pVnode->status != TAOS_VN_STATUS_READY) {
     vDebug("vgId:%d, vnode status is %s, refCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status],
            pVnode->refCount, pVnode);

src/vnode/src/vnodeWrite.c

@@ -17,19 +17,23 @@
 #include "os.h"
 #include "taosmsg.h"
 #include "taoserror.h"
+#include "tglobal.h"
 #include "tqueue.h"
 #include "trpc.h"
 #include "tsdb.h"
 #include "twal.h"
 #include "tsync.h"
+#include "ttimer.h"
 #include "tdataformat.h"
 #include "vnode.h"
 #include "vnodeInt.h"
 #include "syncInt.h"
 #include "tcq.h"
+#include "dnode.h"
 
 #define MAX_QUEUED_MSG_NUM 10000
 
+extern void *  tsDnodeTmr;
 static int32_t (*vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *, void *pCont, SRspRet *);
 static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *);
 static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *);
@@ -37,6 +41,7 @@ static int32_t vnodeProcessDropTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet
 static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *);
 static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *);
 static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspRet *);
+static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite);
 
 void vnodeInitWriteFp(void) {
   vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_SUBMIT]          = vnodeProcessSubmitMsg;
@@ -49,7 +54,7 @@ void vnodeInitWriteFp(void) {
 
 int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam) {
   int32_t    code = 0;
-  SVnodeObj * pVnode = vparam;
+  SVnodeObj *pVnode = vparam;
   SWalHead * pHead = wparam;
   SRspRet *  pRspRet = rparam;
 
@@ -77,8 +82,6 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara
 
     // assign version
     pHead->version = pVnode->version + 1;
-    if (pVnode->delayMs) taosMsleep(pVnode->delayMs);
-
   } else {  // from wal or forward
     // for data from WAL or forward, version may be smaller
     if (pHead->version <= pVnode->version) return 0;
@@ -218,9 +221,10 @@ static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspR
 int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam) {
   SVnodeObj *pVnode = vparam;
   SWalHead * pHead = wparam;
+  int32_t    code = 0;
 
   if (qtype == TAOS_QTYPE_RPC) {
-    int32_t code = vnodeCheckWrite(pVnode);
+    code = vnodeCheckWrite(pVnode);
     if (code != TSDB_CODE_SUCCESS) return code;
   }
 
@@ -237,11 +241,12 @@ int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rpar
 
   if (rparam != NULL) {
     SRpcMsg *pRpcMsg = rparam;
-    pWrite->rpcHandle = pRpcMsg->handle;
-    pWrite->rpcAhandle = pRpcMsg->ahandle;
+    pWrite->rpcMsg = *pRpcMsg;
   }
 
   memcpy(pWrite->pHead, pHead, sizeof(SWalHead) + pHead->len);
+  pWrite->pVnode = pVnode;
+  pWrite->qtype = qtype;
 
   atomic_add_fetch_32(&pVnode->refCount, 1);
 
@@ -251,6 +256,9 @@ int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rpar
     taosMsleep(1);
   }
 
+  code = vnodePerformFlowCtrl(pWrite);
+  if (code != 0) return 0;
+
   vTrace("vgId:%d, write into vwqueue, refCount:%d queued:%d", pVnode->vgId, pVnode->refCount, pVnode->queuedWMsg);
 
   taosWriteQitem(pVnode->wqueue, qtype, pWrite);
@@ -260,9 +268,52 @@ int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rpar
 void vnodeFreeFromWQueue(void *vparam, SVWriteMsg *pWrite) {
   SVnodeObj *pVnode = vparam;
 
-  atomic_sub_fetch_32(&pVnode->queuedWMsg, 1);
-  vTrace("vgId:%d, free from vwqueue, refCount:%d queued:%d", pVnode->vgId, pVnode->refCount, pVnode->queuedWMsg);
+  int32_t queued = atomic_sub_fetch_32(&pVnode->queuedWMsg, 1);
+  vTrace("vgId:%d, msg:%p, app:%p, free from vwqueue, queued:%d", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, queued);
 
   taosFreeQitem(pWrite);
   vnodeRelease(pVnode);
 }
+
+static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) {
+  SVWriteMsg *pWrite = param;
+  SVnodeObj * pVnode = pWrite->pVnode;
+  int32_t     code = TSDB_CODE_VND_SYNCING;
+
+  pWrite->processedCount++;
+  if (pWrite->processedCount > 100) {
+    vError("vgId:%d, msg:%p, failed to process since %s, retry:%d", pVnode->vgId, pWrite, tstrerror(code),
+           pWrite->processedCount);
+    pWrite->processedCount = 1;
+    dnodeSendRpcVWriteRsp(pWrite->pVnode, pWrite, code);
+  } else {
+    code = vnodePerformFlowCtrl(pWrite);
+    if (code == 0) {
+      vDebug("vgId:%d, msg:%p, write into vwqueue after flowctrl, retry:%d", pVnode->vgId, pWrite,
+             pWrite->processedCount);
+      pWrite->processedCount = 0;
+      taosWriteQitem(pVnode->wqueue, pWrite->qtype, pWrite);
+    }
+  }
+}
+
+static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) {
+  SVnodeObj *pVnode = pWrite->pVnode;
+  if (pVnode->flowctrlLevel <= 0) return 0;
+  if (pWrite->qtype != TAOS_QTYPE_RPC) return 0;
+
+  if (tsFlowCtrl == 0) {
+    int32_t ms = pow(2, pVnode->flowctrlLevel + 2);
+    if (ms > 100) ms = 100;
+    vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl for %d ms", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, ms);
+    taosMsleep(ms);
+    return 0;
+  } else {
+    void *unUsed = NULL;
+    taosTmrReset(vnodeFlowCtrlMsgToWQueue, 100, pWrite, tsDnodeTmr, &unUsed);
+
+    vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl, retry:%d", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle,
+           pWrite->processedCount);
+    return TSDB_CODE_VND_ACTION_IN_PROGRESS;
+  }
+}

src/wal/inc/walInt.h

@@ -44,8 +44,8 @@ typedef struct {
   uint64_t version;
   int64_t  fileId;
   int64_t  rid;
+  int64_t  tfd;
   int32_t  vgId;
-  int32_t  fd;
   int32_t  keep;
   int32_t  level;
   int32_t  fsyncPeriod;

src/wal/src/walMgmt.c

@@ -17,6 +17,7 @@
 #include "os.h"
 #include "taoserror.h"
 #include "tref.h"
+#include "tfile.h"
 #include "twal.h"
 #include "walInt.h"
 
@@ -61,7 +62,7 @@ void *walOpen(char *path, SWalCfg *pCfg) {
   }
 
   pWal->vgId = pCfg->vgId;
-  pWal->fd = -1;
+  pWal->tfd = -1;
   pWal->fileId = -1;
   pWal->level = pCfg->walLevel;
   pWal->keep = pCfg->keep;
@@ -124,7 +125,7 @@ void walClose(void *handle) {
 
   SWal *pWal = handle;
   pthread_mutex_lock(&pWal->mutex);
-  taosClose(pWal->fd);
+  tfClose(pWal->tfd);
   pthread_mutex_unlock(&pWal->mutex);
   taosRemoveRef(tsWal.refId, pWal->rid);
 }
@@ -143,7 +144,7 @@ static void walFreeObj(void *wal) {
   SWal *pWal = wal;
   wDebug("vgId:%d, wal:%p is freed", pWal->vgId, pWal);
 
-  taosClose(pWal->fd);
+  tfClose(pWal->tfd);
   pthread_mutex_destroy(&pWal->mutex);
   tfree(pWal);
 }
@@ -172,7 +173,7 @@ static void walFsyncAll() {
   while (pWal) {
     if (walNeedFsync(pWal)) {
       wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->vgId, pWal->level, pWal->fsyncSeq, tsWal.seq);
-      int32_t code = fsync(pWal->fd);
+      int32_t code = tfFsync(pWal->tfd);
       if (code != 0) {
         wError("vgId:%d, file:%s, failed to fsync since %s", pWal->vgId, pWal->name, strerror(code));
       }

src/wal/src/walWrite.c

@@ -18,6 +18,7 @@
 #include "os.h"
 #include "taoserror.h"
 #include "tchecksum.h"
+#include "tfile.h"
 #include "twal.h"
 #include "walInt.h"
 
@@ -36,8 +37,8 @@ int32_t walRenew(void *handle) {
 
   pthread_mutex_lock(&pWal->mutex);
 
-  if (pWal->fd >= 0) {
-    tclose(pWal->fd);
+  if (tfValid(pWal->tfd)) {
+    tfClose(pWal->tfd);
     wDebug("vgId:%d, file:%s, it is closed", pWal->vgId, pWal->name);
   }
 
@@ -49,9 +50,9 @@ int32_t walRenew(void *handle) {
   }
 
   snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, pWal->fileId);
-  pWal->fd = open(pWal->name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
+  pWal->tfd = tfOpenM(pWal->name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
 
-  if (pWal->fd < 0) {
+  if (!tfValid(pWal->tfd)) {
     code = TAOS_SYSTEM_ERROR(errno);
     wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->name, strerror(errno));
   } else {
@@ -67,7 +68,7 @@ void walRemoveOneOldFile(void *handle) {
   SWal *pWal = handle;
   if (pWal == NULL) return;
   if (pWal->keep == TAOS_WAL_KEEP) return;
-  if (pWal->fd <= 0) return;
+  if (!tfValid(pWal->tfd)) return;
 
   pthread_mutex_lock(&pWal->mutex);
 
@@ -113,7 +114,7 @@ int32_t walWrite(void *handle, SWalHead *pHead) {
   int32_t code = 0;
 
   // no wal
-  if (pWal->fd <= 0) return 0;
+  if (!tfValid(pWal->tfd)) return 0;
   if (pWal->level == TAOS_WAL_NOLOG) return 0;
   if (pHead->version <= pWal->version) return 0;
 
@@ -123,12 +124,12 @@ int32_t walWrite(void *handle, SWalHead *pHead) {
 
   pthread_mutex_lock(&pWal->mutex);
 
-  if (taosWrite(pWal->fd, pHead, contLen) != contLen) {
+  if (tfWrite(pWal->tfd, pHead, contLen) != contLen) {
     code = TAOS_SYSTEM_ERROR(errno);
     wError("vgId:%d, file:%s, failed to write since %s", pWal->vgId, pWal->name, strerror(errno));
   } else {
-    wTrace("vgId:%d, write wal, fileId:%" PRId64 " fd:%d hver:%" PRId64 " wver:%" PRIu64 " len:%d", pWal->vgId,
-           pWal->fileId, pWal->fd, pHead->version, pWal->version, pHead->len);
+    wTrace("vgId:%d, write wal, fileId:%" PRId64 " tfd:%" PRId64 " hver:%" PRId64 " wver:%" PRIu64 " len:%d", pWal->vgId,
+           pWal->fileId, pWal->tfd, pHead->version, pWal->version, pHead->len);
     pWal->version = pHead->version;
   }
 
@@ -141,12 +142,12 @@ int32_t walWrite(void *handle, SWalHead *pHead) {
 
 void walFsync(void *handle, bool forceFsync) {
   SWal *pWal = handle;
-  if (pWal == NULL || pWal->fd < 0) return;
+  if (pWal == NULL || !tfValid(pWal->tfd)) return;
 
   if (forceFsync || (pWal->level == TAOS_WAL_FSYNC && pWal->fsyncPeriod == 0)) {
-    wTrace("vgId:%d, file:%s, do fsync", pWal->vgId, pWal->name);
-    if (fsync(pWal->fd) < 0) {
-      wError("vgId:%d, file:%s, fsync failed since %s", pWal->vgId, pWal->name, strerror(errno));
+    wTrace("vgId:%d, fileId:%" PRId64 ", do fsync", pWal->vgId, pWal->fileId);
+    if (tfFsync(pWal->tfd) < 0) {
+      wError("vgId:%d, fileId:%" PRId64 ", fsync failed since %s", pWal->vgId, pWal->fileId, strerror(errno));
     }
   }
 }
@@ -186,8 +187,8 @@ int32_t walRestore(void *handle, void *pVnode, FWalWrite writeFp) {
     // open the existing WAL file in append mode
     pWal->fileId = 0;
     snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, pWal->fileId);
-    pWal->fd = open(pWal->name, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
-    if (pWal->fd < 0) {
+    pWal->tfd = tfOpenM(pWal->name, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
+    if (!tfValid(pWal->tfd)) {
       wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->name, strerror(errno));
       return TAOS_SYSTEM_ERROR(errno);
     }
@@ -211,28 +212,28 @@ int32_t walGetWalFile(void *handle, char *fileName, int64_t *fileId) {
     code = (*fileId == pWal->fileId) ? 0 : 1;
   }
 
-  wTrace("vgId:%d, get wal file, code:%d curId:%" PRId64 " outId:%" PRId64, pWal->vgId, code, pWal->fileId, *fileId);
+  wDebug("vgId:%d, get wal file, code:%d curId:%" PRId64 " outId:%" PRId64, pWal->vgId, code, pWal->fileId, *fileId);
   pthread_mutex_unlock(&(pWal->mutex));
 
   return code;
 }
 
-static void walFtruncate(SWal *pWal, int32_t fd, int64_t offset) {
-  taosFtruncate(fd, offset);
-  fsync(fd);
+static void walFtruncate(SWal *pWal, int64_t tfd, int64_t offset) {
+  tfFtruncate(tfd, offset);
+  tfFsync(tfd);
 }
 
-static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int32_t fd, int64_t *offset) {
+static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int64_t tfd, int64_t *offset) {
   int64_t pos = *offset;
   while (1) {
     pos++;
 
-    if (lseek(fd, pos, SEEK_SET) < 0) {
+    if (tfLseek(tfd, pos, SEEK_SET) < 0) {
       wError("vgId:%d, failed to seek from corrupted wal file since %s", pWal->vgId, strerror(errno));
       return TSDB_CODE_WAL_FILE_CORRUPTED;
     }
 
-    if (taosRead(fd, pHead, sizeof(SWalHead)) <= 0) {
+    if (tfRead(tfd, pHead, sizeof(SWalHead)) <= 0) {
       wError("vgId:%d, read to end of corrupted wal file, offset:%" PRId64, pWal->vgId, pos);
       return TSDB_CODE_WAL_FILE_CORRUPTED;
     }
@@ -259,8 +260,8 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch
     return TAOS_SYSTEM_ERROR(errno);
   }
 
-  int32_t fd = open(name, O_RDWR);
-  if (fd < 0) {
+  int64_t tfd = tfOpen(name, O_RDWR);
+  if (!tfValid(tfd)) {
     wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno));
     tfree(buffer);
     return TAOS_SYSTEM_ERROR(errno);
@@ -273,7 +274,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch
   SWalHead *pHead = buffer;
 
   while (1) {
-    int32_t ret = taosRead(fd, pHead, sizeof(SWalHead));
+    int32_t ret = tfRead(tfd, pHead, sizeof(SWalHead));
     if (ret == 0) break;
 
     if (ret < 0) {
@@ -284,16 +285,16 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch
 
     if (ret < sizeof(SWalHead)) {
       wError("vgId:%d, file:%s, failed to read wal head, ret is %d", pWal->vgId, name, ret);
-      walFtruncate(pWal, fd, offset);
+      walFtruncate(pWal, tfd, offset);
       break;
     }
 
     if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) {
       wError("vgId:%d, file:%s, wal head cksum is messed up, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name,
              pHead->version, pHead->len, offset);
-      code = walSkipCorruptedRecord(pWal, pHead, fd, &offset);
+      code = walSkipCorruptedRecord(pWal, pHead, tfd, &offset);
       if (code != TSDB_CODE_SUCCESS) {
-        walFtruncate(pWal, fd, offset);
+        walFtruncate(pWal, tfd, offset);
         break;
       }
     }
@@ -310,7 +311,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch
       pHead = buffer;
     }
 
-    ret = taosRead(fd, pHead->cont, pHead->len);
+    ret = tfRead(tfd, pHead->cont, pHead->len);
     if (ret < 0) {
       wError("vgId:%d, file:%s, failed to read wal body since %s", pWal->vgId, name, strerror(errno));
       code = TAOS_SYSTEM_ERROR(errno);
@@ -325,14 +326,14 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch
 
     offset = offset + sizeof(SWalHead) + pHead->len;
 
-    wTrace("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d", pWal->vgId,
+    wDebug("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d", pWal->vgId,
            fileId, pHead->version, pWal->version, pHead->len);
 
     pWal->version = pHead->version;
     (*writeFp)(pVnode, pHead, TAOS_QTYPE_WAL, NULL);
   }
 
-  tclose(fd);
+  tfClose(tfd);
   tfree(buffer);
 
   return code;

tests/examples/c/apitest.c

@@ -79,11 +79,11 @@ static int print_result(TAOS_RES* res, int blockFetch) {
   if (blockFetch) {
     int rows = 0;
     while ((rows = taos_fetch_block(res, &row))) {
-      for (int i = 0; i < rows; i++) {
-        char temp[256];
-        taos_print_row(temp, row + i, fields, num_fields);
-        puts(temp);
-      }
+      //for (int i = 0; i < rows; i++) {
+      //  char temp[256];
+      //  taos_print_row(temp, row + i, fields, num_fields);
+      //  puts(temp);
+      //}
       nRows += rows;
     }
   } else {

tests/examples/c/subscribe.c

@@ -19,10 +19,10 @@ void print_result(TAOS_RES* res, int blockFetch) {
   
   if (blockFetch) {
     nRows = taos_fetch_block(res, &row);
-    for (int i = 0; i < nRows; i++) {
-      taos_print_row(buf, row + i, fields, num_fields);
-      puts(buf);
-    }
+    //for (int i = 0; i < nRows; i++) {
+    //  taos_print_row(buf, row + i, fields, num_fields);
+    //  puts(buf);
+    //}
   } else {
     while ((row = taos_fetch_row(res))) {
       taos_print_row(buf, row, fields, num_fields);

tests/pytest/cluster/clusterSetup.py

@@ -31,6 +31,23 @@ class Node:
         self.homeDir = homeDir
         self.conn = Connection("{}@{}".format(username, hostName), connect_kwargs={"password": "{}".format(password)}) 
     
+    def buildTaosd(self):
+        try:
+            self.conn.cd("/root/TDinternal/community")
+            self.conn.run("git checkout develop")
+            self.conn.run("git pull")
+            self.conn.cd("/root/TDinternal")
+            self.conn.run("git checkout develop")
+            self.conn.run("git pull")
+            self.conn.cd("/root/TDinternal/debug")
+            self.conn.run("cmake ..")
+            self.conn.run("make")
+            self.conn.run("make install")
+        except Exception as e:
+            print("Build Taosd error for node %d " % self.index)
+            logging.exception(e)
+            pass
+
     def startTaosd(self):
         try:
             self.conn.run("sudo systemctl start taosd")
@@ -105,9 +122,11 @@ class Node:
 
 class Nodes:
     def __init__(self):
-        self.node1 = Node(1, 'ubuntu', '192.168.1.52', 'node1', 'tbase125!', '/home/ubuntu')
-        self.node2 = Node(2, 'ubuntu', '192.168.1.53', 'node2', 'tbase125!', '/home/ubuntu')
-        self.node3 = Node(3, 'ubuntu', '192.168.1.54', 'node3', 'tbase125!', '/home/ubuntu')
+        self.node1 = Node(1, 'root', '52.151.60.239', 'node1', 'r', '/root/')
+        self.node2 = Node(2, 'root', '52.183.32.246', 'node1', 'r', '/root/')
+        self.node3 = Node(3, 'root', '51.143.46.79', 'node1', 'r', '/root/')
+        self.node4 = Node(4, 'root', '52.183.2.76', 'node1', 'r', '/root/')
+        self.node5 = Node(5, 'root', '13.66.225.87', 'node1', 'r', '/root/')        
 
     def stopAllTaosd(self):
         self.node1.stopTaosd()

tests/pytest/concurrent_inquiry.py

@@ -146,7 +146,7 @@ class ConcurrentInquiry:
             col_list=self.stb_stru_list[tbi-1]
             tag_list=self.stb_tag_list[tbi-1]
             is_stb=1
-        tlist=col_list+tag_list
+        tlist=col_list+tag_list+['abc']            #增加不存在的域'abc'，是否会引起新bug
         con_rand=random.randint(0,len(condition_list))
         func_rand=random.randint(0,len(func_list))
         col_rand=random.randint(0,len(col_list))

tests/pytest/fulltest.sh

@@ -158,6 +158,7 @@ python3 ./test.py -f query/bug1471.py
 python3 ./test.py -f query/bug1874.py
 python3 ./test.py -f query/bug1875.py
 python3 ./test.py -f query/bug1876.py
+python3 ./test.py -f query/bug2218.py 
 
 #stream
 python3 ./test.py -f stream/metric_1.py
@@ -205,7 +206,9 @@ python3 test.py -f query/queryInterval.py
 python3 test.py -f query/queryFillTest.py
 
 # tools
-python3 test.py -f tools/taosdemo.py
+python3 test.py -f tools/taosdemoTest.py
+python3 test.py -f tools/taosdumpTest.py
+python3 test.py -f tools/lowaTest.py
 
 # subscribe
 python3 test.py -f subscribe/singlemeter.py

tests/pytest/insert/restfulInsert.py

@@ -18,10 +18,10 @@ import time
 import argparse
 
 class RestfulInsert:
-    def __init__(self, host, dbname, threads, tables, records, batchSize, tbNamePerfix, outOfOrder):
+    def __init__(self, host, startTimestamp, dbname, threads, tables, records, batchSize, tbNamePerfix, outOfOrder):
         self.header = {'Authorization': 'Basic cm9vdDp0YW9zZGF0YQ=='}
         self.url = "http://%s:6041/rest/sql" % host
-        self.ts = 1500000000000
+        self.ts = startTimestamp
         self.dbname = dbname
         self.numOfThreads = threads
         self.numOfTables = tables
@@ -36,8 +36,10 @@ class RestfulInsert:
         for i in range(tablesPerThread):
             tableID = threadID * tablesPerThread
             name = 'beijing' if tableID % 2 == 0 else 'shanghai'
-            data = "create table %s.%s%d using %s.meters tags(%d, '%s')" % (self.dbname, self.tableNamePerfix, tableID + i, self.dbname, tableID + i, name)
-            requests.post(self.url, data, headers = self.header)
+            data = "create table if not exists %s.%s%d using %s.meters tags(%d, '%s')" % (self.dbname, self.tableNamePerfix, tableID + i, self.dbname, tableID + i, name)
+            response = requests.post(self.url, data, headers = self.header)
+            if response.status_code != 200:
+                    print(response.content)
 
     def insertData(self, threadID):        
         print("thread %d started" % threadID)
@@ -50,14 +52,19 @@ class RestfulInsert:
                 values = []
                 for k in range(self.batchSize):
                     data +=  "(%d, %d, %d, %d)" %  (start + j * self.batchSize + k, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100))                                
-                requests.post(self.url, data, headers = self.header)
+                response = requests.post(self.url, data, headers = self.header)
+                if response.status_code != 200:
+                    print(response.content)
 
     def insertUnlimitedData(self, threadID):        
         print("thread %d started" % threadID)
         tablesPerThread = int (self.numOfTables / self.numOfThreads)
+        
+        count = 0
         while True:
             i = 0
-            start = self.ts
+            start = self.ts  + count * self.batchSize          
+            count = count + 1            
             
             for i in range(tablesPerThread):
                 tableID = i + threadID * tablesPerThread
@@ -65,7 +72,7 @@ class RestfulInsert:
                 data = "insert into %s.%s%d values" % (self.dbname, self.tableNamePerfix, tableID)
                 values = []
                 for k in range(self.batchSize):
-                    values.append("(%d, %d, %d, %d)" %  (start + j * self.batchSize + k, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100)))
+                    values.append("(%d, %d, %d, %d)" %  (start + k, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100)))
                 
                 if(self.outOfOrder == False):
                     for k in range(len(values)):            
@@ -74,14 +81,14 @@ class RestfulInsert:
                     random.shuffle(values)
                     for k in range(len(values)):            
                         data += values[k]                
-                requests.post(self.url, data, headers = self.header)
+                response = requests.post(self.url, data, headers = self.header)
+                if response.status_code != 200:
+                    print(response.content)
 
     def run(self):                
-        data = "drop database if exists %s" % self.dbname
+        data = "create database if not exists %s" % self.dbname
         requests.post(self.url, data, headers = self.header)
-        data = "create database %s" % self.dbname
-        requests.post(self.url, data, headers = self.header)
-        data = "create table %s.meters(ts timestamp, f1 int, f2 int, f3 int) tags(id int, loc nchar(20))" % self.dbname
+        data = "create table if not exists %s.meters(ts timestamp, f1 int, f2 int, f3 int) tags(id int, loc nchar(20))" % self.dbname
         requests.post(self.url, data, headers = self.header)
 
         threads = []
@@ -117,6 +124,13 @@ parser.add_argument(
     default='127.0.0.1',
     type=str,
     help='host name to be connected (default: 127.0.0.1)')
+parser.add_argument(
+    '-S',
+    '--start-timestamp',
+    action='store',
+    default=1500000000000,
+    type=int,
+    help='insert data from timestamp (default: 1500000000000)')
 parser.add_argument(
     '-d',
     '--db-name',
@@ -166,5 +180,5 @@ parser.add_argument(
     help='The order of test data (default: False)')
 
 args = parser.parse_args()
-ri = RestfulInsert(args.host_name, args.db_name, args.number_of_threads, args.number_of_tables, args.number_of_records, args.batch_size, args.table_name_prefix, args.out_of_order)
+ri = RestfulInsert(args.host_name, args.start_timestamp, args.db_name, args.number_of_threads, args.number_of_tables, args.number_of_records, args.batch_size, args.table_name_prefix, args.out_of_order)
 ri.run()
\ No newline at end of file

tests/pytest/query/bug2218.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+        print("==========step1")
+        print("create table && insert data")
+        
+        tdSql.execute("create table mt0 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool)")
+        insertRows = 1000
+        t0 = 1604298064000
+        tdLog.info("insert %d rows" % (insertRows))
+        for i in range(insertRows):
+            ret = tdSql.execute(
+                "insert into mt0 values (%d , %d,%d,%d,%d,%d,%d,%d)" %
+                (t0+i,i%100,i/2,i%100,i%100,i%100,i*1.0,i%2))
+        print("==========step2")
+        print("test col*1*1 desc ")
+        tdSql.query('select c1,c1*1*1,c2*1*1,c3*1*1,c4*1*1,c5*1*1,c6*1*1 from mt0 order by ts  desc limit 2')
+        tdSql.checkData(0,0,99)
+        tdSql.checkData(0,1,0.0)
+        tdSql.checkData(0,2,0.0)
+        tdSql.checkData(0,3,0.0)
+        tdSql.checkData(0,4,0.0)
+        tdSql.checkData(0,5,0.0)
+        tdSql.checkData(0,6,0.0)
+
+        
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file

tests/pytest/query/filterOtherTypes.py

@@ -376,11 +376,9 @@ class TDTestCase:
         tdSql.execute("insert into t1 values(1538548685000, 1) (1538548685001, 2) (1538548685002, 3)")
         tdSql.execute("insert into t2 values(1538548685000, 4) (1538548685001, 5) (1538548685002, 6)")
 
-        tdSql.query("select * from t1 where tag1 like '%g'")
-        tdSql.checkRows(3)
+        tdSql.error("select * from t1 where tag1 like '%g'")        
 
-        tdSql.query("select * from t2 where tag1 like '%g'")
-        tdSql.checkRows(3)
+        tdSql.error("select * from t2 where tag1 like '%g'") 
 
         tdSql.query("select * from meters where tag1 like '%g'")
         tdSql.checkRows(6)
@@ -396,20 +394,16 @@ class TDTestCase:
         tdSql.execute("insert into t5 values(1538548685000, 1) (1538548685001, 2) (1538548685002, 3)")
         tdSql.execute("insert into t6 values(1538548685000, 1) (1538548685001, 2) (1538548685002, 3)")
 
-        tdSql.query("select * from t3 where tag1 like '%京'")
-        tdSql.checkRows(3)
+        tdSql.error("select * from t3 where tag1 like '%京'")        
 
-        tdSql.query("select * from t4 where tag1 like '%京'")
-        tdSql.checkRows(3)
+        tdSql.error("select * from t4 where tag1 like '%京'")        
 
         tdSql.query("select * from meters1 where tag1 like '%京'")
         tdSql.checkRows(6)
         
-        tdSql.query("select * from t5 where tag1 like '%g'")
-        tdSql.checkRows(3)
+        tdSql.error("select * from t5 where tag1 like '%g'")        
 
-        tdSql.query("select * from t6 where tag1 like '%g'")
-        tdSql.checkRows(3)
+        tdSql.error("select * from t6 where tag1 like '%g'")        
 
         tdSql.query("select * from meters1 where tag1 like '%g'")
         tdSql.checkRows(6)

tests/pytest/query/queryInterval.py

@@ -34,28 +34,42 @@ class TDTestCase:
         tdSql.execute("insert into t1 using st tags('shanghai') values(%d, 220) (%d, 221) (%d, 225) (%d, 228) (%d, 222)" 
                         % (self.ts, self.ts + 2000000000, self.ts + 4000000000, self.ts + 5000000000, self.ts + 7000000000))             
                 
+
         tdSql.query("select avg(voltage) from st interval(1n)")
-        tdSql.checkRows(4)
-        tdSql.checkData(0, 1, 220.0)
-        tdSql.checkData(1, 1, 222.33333333333334)
-        tdSql.checkData(2, 1, 227.0)
-        tdSql.checkData(3, 1, 222.0)
+        tdSql.checkRows(3)        
+        tdSql.checkData(0, 0, "2020-07-01 00:00:00")
+        tdSql.checkData(0, 1, 221.4)        
+        tdSql.checkData(1, 0, "2020-08-01 00:00:00")
+        tdSql.checkData(1, 1, 227.0)        
+        tdSql.checkData(2, 0, "2020-09-01 00:00:00")
+        tdSql.checkData(2, 1, 222.0)
         
         tdSql.query("select avg(voltage) from st interval(1n, 15d)")
         tdSql.checkRows(4)
+        tdSql.checkData(0, 0, "2020-06-16 00:00:00")
         tdSql.checkData(0, 1, 220.333333)
+        tdSql.checkData(1, 0, "2020-07-16 00:00:00")
         tdSql.checkData(1, 1, 224.666666)
+        tdSql.checkData(2, 0, "2020-08-16 00:00:00")
         tdSql.checkData(2, 1, 225.0)
+        tdSql.checkData(3, 0, "2020-09-16 00:00:00")
         tdSql.checkData(3, 1, 222.0)
 
         tdSql.query("select avg(voltage) from st interval(1n, 15d) group by loc")
         tdSql.checkRows(7)
+        tdSql.checkData(0, 0, "2020-06-16 00:00:00")
         tdSql.checkData(0, 1, 220.5)
+        tdSql.checkData(1, 0, "2020-07-16 00:00:00")
         tdSql.checkData(1, 1, 226.5)
+        tdSql.checkData(2, 0, "2020-08-16 00:00:00")
         tdSql.checkData(2, 1, 222.0)
+        tdSql.checkData(3, 0, "2020-06-16 00:00:00")
         tdSql.checkData(3, 1, 220.0)
+        tdSql.checkData(4, 0, "2020-07-16 00:00:00")
         tdSql.checkData(4, 1, 221.0)
+        tdSql.checkData(5, 0, "2020-08-16 00:00:00")
         tdSql.checkData(5, 1, 226.5)
+        tdSql.checkData(6, 0, "2020-09-16 00:00:00")
         tdSql.checkData(6, 1, 222.0)
 
     def stop(self):

tests/pytest/query/queryJoin.py

@@ -141,6 +141,42 @@ class TDTestCase:
         tdSql.query("select * from meters1, meters3 where meters1.ts = meters3.ts and meters1.tag1 = meters3.tag1")
         tdSql.checkRows(0)
 
+        tdSql.execute("create table join_mt0(ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) tags(t1 int, t2 binary(12))")
+        tdSql.execute("create table join_mt1(ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) tags(t1 int, t2 binary(12), t3 int)")
+
+        ts = 1538548685000
+        for i in range(3):
+            tdSql.execute("create table join_tb%d using join_mt0 tags(%d, 'abc')" % (i, i)) 
+            sql = "insert into join_tb%d values" % i
+            for j in range(500):
+                val = j % 100
+                sql += "(%d, %d, %f, %d, %d, %d, %f, %d, 'binary%d', 'nchar%d')" % (ts + j, val, val * 1.0, val, val, val, val * 1.0, val % 2,  val, val)
+            tdSql.execute(sql)
+            sql = "insert into join_tb%d values" % i
+            for j in range(500, 1000):
+                val = j % 100
+                sql += "(%d, %d, %f, %d, %d, %d, %f, %d, 'binary%d', 'nchar%d')" % (ts + 500 + j, val, val * 1.0, val, val, val, val * 1.0, val % 2,  val, val)
+            tdSql.execute(sql)
+        
+        for i in range(3):
+            tdSql.execute("create table join_1_tb%d using join_mt1 tags(%d, 'abc%d', %d)" % (i, i, i, i))            
+            sql = "insert into join_1_tb%d values" % i
+            for j in range(500):
+                val = j % 100
+                sql += "(%d, %d, %f, %d, %d, %d, %f, %d, 'binary%d', 'nchar%d')" % (ts + j, val, val * 1.0, val, val, val, val * 1.0, val % 2,  val, val)
+            tdSql.execute(sql)
+            sql = "insert into join_1_tb%d values" % i
+            for j in range(500, 1000):
+                val = j % 100
+                sql += "(%d, %d, %f, %d, %d, %d, %f, %d, 'binary%d', 'nchar%d')" % (ts + 500 + j, val, val * 1.0, val, val, val, val * 1.0, val % 2,  val, val)
+            tdSql.execute(sql)
+
+        tdSql.error("select count(join_mt0.c1), sum(join_mt1.c2), first(join_mt0.c5), last(join_mt1.c7) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1 order by join_mt0.ts desc")
+        tdSql.error("select count(join_mt0.c1), first(join_mt0.c1)-first(join_mt1.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts")
+        tdSql.error("select count(join_mt0.c1), first(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2 order by join_mt0.t1 desc slimit 3")
+        tdSql.error("select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2, join_mt1.t1 order by join_mt0.ts desc, join_mt1.ts asc limit 10;")
+        tdSql.error("select join_mt1.c1,join_mt0.c1 from join_mt1,join_mt0 where join_mt1.ts = join_mt0.ts and join_mt1.t1 = join_mt0.t1 order by t")
+
 
     def stop(self):
         tdSql.close()

tests/pytest/tools/taosdemo.py → tests/pytest/tools/taosdemoTest.py

@@ -55,6 +55,12 @@ class TDTestCase:
         tdSql.query("select count(*) from meters")
         tdSql.checkData(0, 0, self.numberOfTables * self.numberOfRecords)
 
+        tdSql.query("select sum(f1) from test.meters interval(1h) sliding(30m)")
+        tdSql.checkRows(2)
+
+        tdSql.query("select apercentile(f1, 1) from test.meters interval(10s)")
+        tdSql.checkRows(11)
+
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

tests/pytest/tools/taosdumpTest.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import os
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+        
+        self.ts = 1538548685000
+        self.numberOfTables = 10000
+        self.numberOfRecords = 100
+    
+    def run(self):
+        tdSql.prepare()
+
+        tdSql.execute("create table st(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))")
+        tdSql.execute("create table t1 using st tags(1, 'beijing')")        
+        sql = "insert into t1 values"
+        currts = self.ts
+        for i in range(100):
+            sql += "(%d, %d, 'nchar%d')" % (currts + i, i % 100, i % 100)
+        tdSql.execute(sql)
+
+        tdSql.execute("create table t2 using st tags(2, 'shanghai')")        
+        sql = "insert into t2 values"
+        currts = self.ts
+        for i in range(100):
+            sql += "(%d, %d, 'nchar%d')" % (currts + i, i % 100, i % 100)
+        tdSql.execute(sql)
+
+        os.system("taosdump --databases db -o /tmp")
+        
+        tdSql.execute("drop database db")
+        tdSql.query("show databases")
+        tdSql.checkRows(0)
+        
+        os.system("taosdump -i /tmp")
+
+        tdSql.query("show databases")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 'db')
+        
+        tdSql.execute("use db")
+        tdSql.query("show stables")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 'st')
+
+        tdSql.query("show tables")
+        tdSql.checkRows(2)
+        tdSql.checkData(0, 0, 't2')
+        tdSql.checkData(1, 0, 't1')
+
+        tdSql.query("select * from t1")
+        tdSql.checkRows(100)
+        for i in range(100):
+            tdSql.checkData(i, 1, i)
+            tdSql.checkData(i, 2, "nchar%d" % i)
+
+        tdSql.query("select * from t2")
+        tdSql.checkRows(100)
+        for i in range(100):
+            tdSql.checkData(i, 1, i)
+            tdSql.checkData(i, 2, "nchar%d" % i)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file

tests/script/basicSuite.sim

@@ -14,7 +14,6 @@ run general/table/vgroup.sim
 run general/user/authority.sim
 run general/vector/metrics_mix.sim
 run general/vector/table_field.sim
-run general/user/authority.sim
 run general/tag/set.sim
 run general/table/delete_writing.sim
 run general/stable/disk.sim

tests/script/general/wal/sync.sim

@@ -82,6 +82,7 @@ restful d1 table_rest 1591772800 30000
 restful d1 table_rest 1591872800 30000
 restful d1 table_rest 1591972800 30000
 
+sleep 1000
 sql select * from table_rest;
 print rows: $rows
 if $rows != 300000 then

tests/script/tmp/mnodes.sim

@@ -20,6 +20,14 @@ system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20000
 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 20000
 system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 20000
 
+system sh/cfg.sh -n dnode1 -c minTablesPerVnode -v 1000
+system sh/cfg.sh -n dnode2 -c minTablesPerVnode -v 1000
+system sh/cfg.sh -n dnode3 -c minTablesPerVnode -v 1000
+
+system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 20
+system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 20
+system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 20
+
 system sh/cfg.sh -n dnode1 -c replica -v 3
 system sh/cfg.sh -n dnode2 -c replica -v 3
 system sh/cfg.sh -n dnode3 -c replica -v 3

tests/script/unique/cluster/flowctrl.sim

+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3
+
+system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
+
+system sh/cfg.sh -n dnode1 -c http -v 0
+system sh/cfg.sh -n dnode2 -c http -v 0
+system sh/cfg.sh -n dnode3 -c http -v 0
+
+system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20000
+system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 20000
+system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 20000
+
+system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 20
+system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 20
+system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 20
+
+system sh/cfg.sh -n dnode1 -c replica -v 3
+system sh/cfg.sh -n dnode2 -c replica -v 3
+system sh/cfg.sh -n dnode3 -c replica -v 3
+
+print ============== deploy
+
+system sh/exec.sh -n dnode1 -s start 
+sleep 5001
+sql connect
+
+sql create dnode $hostname2
+sql create dnode $hostname3
+system sh/exec.sh -n dnode2 -s start 
+system sh/exec.sh -n dnode3 -s start 
+
+print  =============== step1
+$x = 0
+show1: 
+	$x = $x + 1
+	sleep 2000
+	if $x == 5 then
+	  return -1
+	endi
+sql show mnodes -x show1
+$mnode1Role = $data2_1
+print mnode1Role $mnode1Role
+$mnode2Role = $data2_2
+print mnode2Role $mnode2Role
+$mnode3Role = $data2_3
+print mnode3Role $mnode3Role
+
+if $mnode1Role != master then
+  goto show1
+endi
+if $mnode2Role != slave then
+  goto show1
+endi
+if $mnode3Role != slave then
+  goto show1
+endi
+
+print  =============== step2
+
+sql create database db replica 3
+sql use db
+sql create table tb (ts timestamp, test int)
+
+$x = 0
+while $x < 100
+  $ms = $x . s
+  sql insert into tb values (now + $ms , $x )
+  $x = $x + 1
+endw 
+
+print  =============== step3
+sleep 3000
+
+system sh/exec.sh -n dnode1 -s stop  -x SIGINT
+system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+system sh/exec.sh -n dnode3 -s stop  -x SIGINT
+
+print  =============== step4
+sleep 5000
+system sh/exec.sh -n dnode1 -s start 
+system sh/exec.sh -n dnode2 -s start 
+system sh/exec.sh -n dnode3 -s start 
+
+print  =============== step5
+sleep 8000
+while $x < 200
+  $ms = $x . s
+  sql insert into tb values (now + $ms , $x )
+  $x = $x + 1
+endw 
+
+print  =============== step6
+system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+sleep 3000
+while $x < 300
+  $ms = $x . s
+  sql insert into tb values (now + $ms , $x )
+  $x = $x + 1
+endw 
+
+system sh/exec.sh -n dnode2 -s start 
+
+sleep 6000
+print  =============== step7
+while $x < 400
+  $ms = $x . s
+  sql insert into tb values (now + $ms , $x )
+  $x = $x + 1
+  sleep 1
+endw 
+
+print  =============== step8
+sql select * from tb
+print rows $rows
+if $rows != 400 then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop  -x SIGINT
+system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+system sh/exec.sh -n dnode3 -s stop  -x SIGINT
\ No newline at end of file

tests/test-all.sh

@@ -17,9 +17,9 @@ function runSimCaseOneByOne {
         echo -e "${GREEN}$case success${NC}" | tee -a out.log || \
         echo -e "${RED}$case failed${NC}" | tee -a out.log
       out_log=`tail -1 out.log  `
-      if [[ $out_log =~ 'failed' ]];then
-        exit 8
-      fi
+      # if [[ $out_log =~ 'failed' ]];then
+      #   exit 8
+      # fi
       end_time=`date +%s`
       echo execution time of $case was `expr $end_time - $start_time`s. | tee -a out.log
     fi
@@ -42,9 +42,9 @@ function runPyCaseOneByOne {
           echo -e "${RED}$case failed${NC}" | tee -a pytest-out.log
         end_time=`date +%s`
         out_log=`tail -1 pytest-out.log  `
-        if [[ $out_log =~ 'failed' ]];then
-          exit 8
-        fi
+        # if [[ $out_log =~ 'failed' ]];then
+        #   exit 8
+        # fi
         echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log
       else
         $line > /dev/null 2>&1

tests/test/c/CMakeLists.txt

@@ -31,8 +31,8 @@ IF (TD_LINUX)
   #add_executable(createTablePerformance createTablePerformance.c)
   #target_link_libraries(createTablePerformance taos_static tutil common pthread)
 
-  #add_executable(createNormalTable createNormalTable.c)
-  #target_link_libraries(createNormalTable taos_static tutil common pthread)
+  add_executable(createNormalTable createNormalTable.c)
+  target_link_libraries(createNormalTable taos_static tutil common pthread)
 
   #add_executable(queryPerformance queryPerformance.c)
   #target_link_libraries(queryPerformance taos_static tutil common pthread)
@@ -45,5 +45,8 @@ IF (TD_LINUX)
 
   #add_executable(invalidTableId invalidTableId.c)
   #target_link_libraries(invalidTableId taos_static tutil common pthread)
+
+  add_executable(hashIterator hashIterator.c)
+  target_link_libraries(hashIterator taos_static tutil common pthread)
 ENDIF()
 

tests/test/c/hashIterator.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taos.h"
+#include "tulog.h"
+#include "tutil.h"
+#include "hash.h"
+
+typedef struct HashTestRow {
+  int32_t keySize;
+  char    key[100];
+} HashTestRow;
+
+int main(int argc, char *argv[]) {
+  _hash_fn_t hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
+  void *     hashHandle = taosHashInit(100, hashFp, true, HASH_ENTRY_LOCK);
+
+  pPrint("insert 3 rows to hash");
+  for (int32_t t = 0; t < 3; ++t) {
+    HashTestRow row = {0};
+    row.keySize = sprintf(row.key, "0.db.st%d", t);
+
+    taosHashPut(hashHandle, row.key, row.keySize, &row, sizeof(HashTestRow));
+  }
+
+  pPrint("start iterator");
+  HashTestRow *row = taosHashIterate(hashHandle, NULL);
+  while (row) {
+    pPrint("drop key:%s", row->key);
+    taosHashRemove(hashHandle, row->key, row->keySize);
+
+    pPrint("get rows from hash");
+    for (int32_t t = 0; t < 3; ++t) {
+      HashTestRow r = {0};
+      r.keySize = sprintf(r.key, "0.db.st%d", t);
+
+      void *result = taosHashGet(hashHandle, r.key, r.keySize);
+      pPrint("get key:%s result:%p", r.key, result);
+    }
+
+    //Before getting the next iterator, the object just deleted can be obtained
+    row = taosHashIterate(hashHandle, row);
+  }
+
+  pPrint("stop iterator");
+  taosHashCancelIterate(hashHandle, row);
+
+  pPrint("get rows from hash");
+  for (int32_t t = 0; t < 3; ++t) {
+    HashTestRow r = {0};
+    r.keySize = sprintf(r.key, "0.db.st%d", t);
+
+    void *result = taosHashGet(hashHandle, r.key, r.keySize);
+    pPrint("get key:%s result:%p", r.key, result);
+  }
+
+  return 0;
+}
\ No newline at end of file