Merge branch 'develop' into test/jenkins

packaging/deb/makedeb.sh

@@ -48,7 +48,7 @@ cp ${compile_dir}/../packaging/deb/taosd            ${pkg_dir}${install_home_pat
 cp ${compile_dir}/../packaging/tools/post.sh        ${pkg_dir}${install_home_path}/script
 cp ${compile_dir}/../packaging/tools/preun.sh       ${pkg_dir}${install_home_path}/script
 cp ${compile_dir}/build/bin/taosdemo                ${pkg_dir}${install_home_path}/bin
-cp ${compile_dir}/build/bin/taosdump                ${pkg_dir}${install_home_path}/bin
+#cp ${compile_dir}/build/bin/taosdump                ${pkg_dir}${install_home_path}/bin
 cp ${compile_dir}/build/bin/taosd                   ${pkg_dir}${install_home_path}/bin
 cp ${compile_dir}/build/bin/taos                    ${pkg_dir}${install_home_path}/bin
 cp ${compile_dir}/build/lib/${libfile}              ${pkg_dir}${install_home_path}/driver 

packaging/rpm/tdengine.spec

@@ -58,7 +58,7 @@ cp %{_compiledir}/../packaging/tools/preun.sh       %{buildroot}%{homepath}/scri
 cp %{_compiledir}/build/bin/taos                    %{buildroot}%{homepath}/bin
 cp %{_compiledir}/build/bin/taosd                   %{buildroot}%{homepath}/bin
 cp %{_compiledir}/build/bin/taosdemo                %{buildroot}%{homepath}/bin
-cp %{_compiledir}/build/bin/taosdump                %{buildroot}%{homepath}/bin
+#cp %{_compiledir}/build/bin/taosdump                %{buildroot}%{homepath}/bin
 cp %{_compiledir}/build/lib/${libfile}              %{buildroot}%{homepath}/driver
 cp %{_compiledir}/../src/inc/taos.h                 %{buildroot}%{homepath}/include
 cp %{_compiledir}/../src/inc/taoserror.h            %{buildroot}%{homepath}/include
@@ -135,7 +135,7 @@ if [ $1 -eq 0 ];then
     ${csudo} rm -f ${bin_link_dir}/taos       || :
     ${csudo} rm -f ${bin_link_dir}/taosd      || :
     ${csudo} rm -f ${bin_link_dir}/taosdemo   || :
-    ${csudo} rm -f ${bin_link_dir}/taosdump   || :
+    #${csudo} rm -f ${bin_link_dir}/taosdump   || :
     ${csudo} rm -f ${cfg_link_dir}/*          || :
     ${csudo} rm -f ${inc_link_dir}/taos.h     || :
     ${csudo} rm -f ${inc_link_dir}/taoserror.h     || :

packaging/tools/makeclient.sh

@@ -45,7 +45,8 @@ if [ "$osType" != "Darwin" ]; then
     strip ${build_dir}/bin/taos
     bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh"
   else  
-    bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh"
+    #bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh"
+    bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh"
   fi
   lib_files="${build_dir}/lib/libtaos.so.${version}"
 else

packaging/tools/makepkg.sh

@@ -36,7 +36,8 @@ if [ "$pagMode" == "lite" ]; then
   strip ${build_dir}/bin/taos
   bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh"
 else 
-  bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh"
+  #bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh"
+  bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh"
 fi
 
 lib_files="${build_dir}/lib/libtaos.so.${version}"

src/client/inc/tsclient.h

@@ -330,6 +330,7 @@ typedef struct STscObj {
   char               writeAuth : 1;
   char               superAuth : 1;
   uint32_t           connId;
+  uint64_t           rid;      // ref ID returned by taosAddRef
   struct SSqlObj *   pHb;
   struct SSqlObj *   sqlList;
   struct SSqlStream *streamList;
@@ -348,7 +349,7 @@ typedef struct SSqlObj {
   void            *signature;
   pthread_t        owner;        // owner of sql object, by which it is executed
   STscObj         *pTscObj;
-  void            *pRpcCtx;
+  int64_t          rpcRid;
   void            (*fp)();
   void            (*fetchFp)();
   void            *param;

src/client/src/tscSQLParser.c

@@ -16,6 +16,7 @@
 #define _BSD_SOURCE
 #define _XOPEN_SOURCE 500
 #define _DEFAULT_SOURCE
+#define _GNU_SOURCE
 
 #include "os.h"
 #include "qAst.h"
@@ -2037,7 +2038,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
             SColumnIndex index = {.tableIndex = j, .columnIndex = i};
 
             char name[TSDB_COL_NAME_LEN] = {0};
-            SStrToken t = {.z = pSchema->name, .n = (uint32_t)strnlen(pSchema->name, TSDB_COL_NAME_LEN)};
+            SStrToken t = {.z = pSchema[i].name, .n = (uint32_t)strnlen(pSchema[i].name, TSDB_COL_NAME_LEN)};
             setResultColName(name, pItem, cvtFunc.originFuncId, &t);
 
             if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[index.columnIndex], cvtFunc, name, colIndex, &index, finalResult) != 0) {

src/client/src/tscServer.c

@@ -182,27 +182,23 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
     int32_t waitingDuring = tsShellActivityTimer * 500;
     tscDebug("%p send heartbeat in %dms", pSql, waitingDuring);
 
-    taosTmrReset(tscProcessActivityTimer, waitingDuring, pObj, tscTmr, &pObj->pTimer);
+    taosTmrReset(tscProcessActivityTimer, waitingDuring, (void *)pObj->rid, tscTmr, &pObj->pTimer);
   } else {
     tscDebug("%p start to close tscObj:%p, not send heartbeat again", pSql, pObj);
   }
 }
 
 void tscProcessActivityTimer(void *handle, void *tmrId) {
-  STscObj *pObj = (STscObj *)handle;
-
-  int ret = taosAcquireRef(tscRefId, pObj);
-  if (ret < 0) {
-    tscTrace("%p failed to acquire TSC obj, reason:%s", pObj, tstrerror(ret));
-    return;
-  }
+  int64_t rid = (int64_t) handle;
+  STscObj *pObj = taosAcquireRef(tscRefId, rid);
+  if (pObj == NULL) return; 
 
   SSqlObj* pHB = pObj->pHb;
 
   void** p = taosCacheAcquireByKey(tscObjCache, &pHB, sizeof(TSDB_CACHE_PTR_TYPE));
   if (p == NULL) {
     tscWarn("%p HB object has been released already", pHB);
-    taosReleaseRef(tscRefId, pObj);
+    taosReleaseRef(tscRefId, pObj->rid);
     return;
   }
 
@@ -216,7 +212,7 @@ void tscProcessActivityTimer(void *handle, void *tmrId) {
     tscError("%p failed to sent HB to server, reason:%s", pHB, tstrerror(code));
   }
 
-  taosReleaseRef(tscRefId, pObj);
+  taosReleaseRef(tscRefId, rid);
 }
 
 int tscSendMsgToServer(SSqlObj *pSql) {
@@ -241,7 +237,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
       .pCont   = pMsg,
       .contLen = pSql->cmd.payloadLen,
       .ahandle = pSql,
-      .handle  = &pSql->pRpcCtx,
+      .handle  = NULL,
       .code    = 0
   };
 
@@ -249,7 +245,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
   // Otherwise, the pSql object may have been released already during the response function, which is
   // processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely
   // cause crash.
-  rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg);
+  pSql->rpcRid = rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -269,7 +265,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
   SSqlCmd *pCmd = &pSql->cmd;
 
   assert(*pSql->self == pSql);
-  pSql->pRpcCtx = NULL;
+  pSql->rpcRid = -1;
 
   if (pObj->signature != pObj) {
     tscDebug("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature);
@@ -1722,8 +1718,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
 
   pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscMetaCache, pTableMetaInfo->name,
       strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer * 1000);
-  
-  // todo handle out of memory case
+
   if (pTableMetaInfo->pTableMeta == NULL) {
     free(pTableMeta);
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
@@ -2026,7 +2021,7 @@ int tscProcessConnectRsp(SSqlObj *pSql) {
   createHBObj(pObj);
 
   //launch a timer to send heartbeat to maintain the connection and send status to mnode
-  taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer);
+  taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, (void *)pObj->rid, tscTmr, &pObj->pTimer);
 
   return 0;
 }

src/client/src/tscSql.c

@@ -161,7 +161,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
   registerSqlObj(pSql);
   tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg);
 
-  taosAddRef(tscRefId, pObj);
+  pObj->rid = taosAddRef(tscRefId, pObj);
   return pSql;
 }
 
@@ -279,9 +279,9 @@ void taos_close(TAOS *taos) {
 
   SSqlObj* pHb = pObj->pHb;
   if (pHb != NULL && atomic_val_compare_exchange_ptr(&pObj->pHb, pHb, 0) == pHb) {
-    if (pHb->pRpcCtx != NULL) {  // wait for rsp from dnode
-      rpcCancelRequest(pHb->pRpcCtx);
-      pHb->pRpcCtx = NULL;
+    if (pHb->rpcRid > 0) {  // wait for rsp from dnode
+      rpcCancelRequest(pHb->rpcRid);
+      pHb->rpcRid = -1;
     }
 
     tscDebug("%p HB is freed", pHb);
@@ -298,7 +298,7 @@ void taos_close(TAOS *taos) {
 
   tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn);
 
-  taosRemoveRef(tscRefId, pObj);
+  taosRemoveRef(tscRefId, pObj->rid);
 }
 
 void waitForQueryRsp(void *param, TAOS_RES *tres, int code) {
@@ -748,9 +748,9 @@ static void tscKillSTableQuery(SSqlObj *pSql) {
     assert(pSubObj->self == (SSqlObj**) p);
 
     pSubObj->res.code = TSDB_CODE_TSC_QUERY_CANCELLED;
-    if (pSubObj->pRpcCtx != NULL) {
-      rpcCancelRequest(pSubObj->pRpcCtx);
-      pSubObj->pRpcCtx = NULL;
+    if (pSubObj->rpcRid > 0) {
+      rpcCancelRequest(pSubObj->rpcRid);
+      pSubObj->rpcRid = -1;
     }
 
     tscQueueAsyncRes(pSubObj);
@@ -775,7 +775,7 @@ void taos_stop_query(TAOS_RES *res) {
   SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
 
   if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
-    assert(pSql->pRpcCtx == NULL);
+    assert(pSql->rpcRid <= 0);
     tscKillSTableQuery(pSql);
   } else {
     if (pSql->cmd.command < TSDB_SQL_LOCAL) {
@@ -784,9 +784,9 @@ void taos_stop_query(TAOS_RES *res) {
        * reset and freed in the processMsgFromServer function, and causes the invalid
        * write problem for rpcCancelRequest.
        */
-      if (pSql->pRpcCtx != NULL) {
-        rpcCancelRequest(pSql->pRpcCtx);
-        pSql->pRpcCtx = NULL;
+      if (pSql->rpcRid > 0) {
+        rpcCancelRequest(pSql->rpcRid);
+        pSql->rpcRid = -1;
       }
 
       tscQueueAsyncRes(pSql);

src/client/src/tscSystem.c

@@ -36,7 +36,7 @@ void *  tscTmr;
 void *  tscQhandle;
 void *  tscCheckDiskUsageTmr;
 int     tsInsertHeadSize;
-int     tscRefId;
+int     tscRefId = -1;
 
 int tscNumOfThreads;
 

src/client/src/tscUtil.c

@@ -376,7 +376,7 @@ void tscFreeRegisteredSqlObj(void *pSql) {
   tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", *p, pTscObj, ref);
   if (ref == 0) {
     tscDebug("%p all sqlObj freed, free tscObj:%p", *p, pTscObj);
-    taosRemoveRef(tscRefId, pTscObj);
+    taosRemoveRef(tscRefId, pTscObj->rid);
   }
 }
 

src/cq/src/cqMain.c

@@ -39,16 +39,16 @@
 #define cTrace(...) { if (cqDebugFlag & DEBUG_TRACE) { taosPrintLog("CQ  ", cqDebugFlag, __VA_ARGS__); }}
 
 typedef struct {
-  int      vgId;
+  int32_t  vgId;
   char     user[TSDB_USER_LEN];
   char     pass[TSDB_PASSWORD_LEN];
   char     db[TSDB_DB_NAME_LEN];
   FCqWrite cqWrite;
   void    *ahandle;
-  int      num;      // number of continuous streams
+  int32_t  num;      // number of continuous streams
   struct SCqObj *pHead;
   void    *dbConn;
-  int      master;
+  int32_t  master;
   void    *tmrCtrl;
   pthread_mutex_t mutex;
 } SCqContext;
@@ -57,7 +57,7 @@ typedef struct SCqObj {
   tmr_h          tmrId;
   uint64_t       uid;
   int32_t        tid;      // table ID
-  int            rowSize;  // bytes of a row
+  int32_t        rowSize;  // bytes of a row
   char *         sqlStr;   // SQL string
   STSchema *     pSchema;  // pointer to schema array
   void *         pStream;
@@ -175,7 +175,7 @@ void cqStop(void *handle) {
   pthread_mutex_unlock(&pContext->mutex);
 }
 
-void *cqCreate(void *handle, uint64_t uid, int tid, char *sqlStr, STSchema *pSchema) {
+void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *pSchema) {
   SCqContext *pContext = handle;
 
   SCqObj *pObj = calloc(sizeof(SCqObj), 1);
@@ -237,7 +237,7 @@ void cqDrop(void *handle) {
   pthread_mutex_unlock(&pContext->mutex);
 }
 
-static void doCreateStream(void *param, TAOS_RES *result, int code) {
+static void doCreateStream(void *param, TAOS_RES *result, int32_t code) {
   SCqObj* pObj = (SCqObj*)param;
   SCqContext* pContext = pObj->pContext;
   SSqlObj* pSql = (SSqlObj*)result;
@@ -288,7 +288,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
 
   cDebug("vgId:%d, id:%d CQ:%s stream result is ready", pContext->vgId, pObj->tid, pObj->sqlStr);
 
-  int size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_DATA_ROW_HEAD_SIZE + pObj->rowSize;
+  int32_t size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_DATA_ROW_HEAD_SIZE + pObj->rowSize;
   char *buffer = calloc(size, 1);
 
   SWalHead   *pHead = (SWalHead *)buffer;

src/dnode/inc/dnodeMPeer.h

@@ -20,11 +20,11 @@
 extern "C" {
 #endif
 
-int32_t dnodeInitMnodePeer();
-void    dnodeCleanupMnodePeer();
-int32_t dnodeAllocateMnodePqueue();
-void    dnodeFreeMnodePqueue();
-void    dnodeDispatchToMnodePeerQueue(SRpcMsg *pMsg);
+int32_t dnodeInitMPeer();
+void    dnodeCleanupMPeer();
+int32_t dnodeAllocateMPeerQueue();
+void    dnodeFreeMPeerQueue();
+void    dnodeDispatchToMPeerQueue(SRpcMsg *pMsg);
 
 #ifdef __cplusplus
 }

src/dnode/inc/dnodeMRead.h

@@ -20,11 +20,11 @@
 extern "C" {
 #endif
 
-int32_t dnodeInitMnodeRead();
-void    dnodeCleanupMnodeRead();
-int32_t dnodeAllocateMnodeRqueue();
-void    dnodeFreeMnodeRqueue();
-void    dnodeDispatchToMnodeReadQueue(SRpcMsg *rpcMsg);
+int32_t dnodeInitMRead();
+void    dnodeCleanupMRead();
+int32_t dnodeAllocMReadQueue();
+void    dnodeFreeMReadQueue();
+void    dnodeDispatchToMReadQueue(SRpcMsg *rpcMsg);
 
 #ifdef __cplusplus
 }

src/dnode/inc/dnodeMWrite.h

@@ -20,11 +20,11 @@
 extern "C" {
 #endif
 
-int32_t dnodeInitMnodeWrite();
-void    dnodeCleanupMnodeWrite();
-int32_t dnodeAllocateMnodeWqueue();
-void    dnodeFreeMnodeWqueue();
-void    dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg);
+int32_t dnodeInitMWrite();
+void    dnodeCleanupMWrite();
+int32_t dnodeAllocMWritequeue();
+void    dnodeFreeMWritequeue();
+void    dnodeDispatchToMWriteQueue(SRpcMsg *pMsg);
 
 #ifdef __cplusplus
 }

src/dnode/src/dnodeMPeer.c

@@ -35,44 +35,44 @@ typedef struct {
 typedef struct {
   int32_t curNum;
   int32_t maxNum;
-  SMPeerWorker *peerWorker;
+  SMPeerWorker *worker;
 } SMPeerWorkerPool;
 
-static SMPeerWorkerPool tsMPeerPool;
+static SMPeerWorkerPool tsMPeerWP;
 static taos_qset        tsMPeerQset;
 static taos_queue       tsMPeerQueue;
 
-static void *dnodeProcessMnodePeerQueue(void *param);
+static void *dnodeProcessMPeerQueue(void *param);
 
-int32_t dnodeInitMnodePeer() {
+int32_t dnodeInitMPeer() {
   tsMPeerQset = taosOpenQset();
   
-  tsMPeerPool.maxNum = 1;
-  tsMPeerPool.curNum = 0;
-  tsMPeerPool.peerWorker = (SMPeerWorker *)calloc(sizeof(SMPeerWorker), tsMPeerPool.maxNum);
+  tsMPeerWP.maxNum = 1;
+  tsMPeerWP.curNum = 0;
+  tsMPeerWP.worker = (SMPeerWorker *)calloc(sizeof(SMPeerWorker), tsMPeerWP.maxNum);
 
-  if (tsMPeerPool.peerWorker == NULL) return -1;
-  for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) {
-    SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
+  if (tsMPeerWP.worker == NULL) return -1;
+  for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) {
+    SMPeerWorker *pWorker = tsMPeerWP.worker + i;
     pWorker->workerId = i;
     dDebug("dnode mpeer worker:%d is created", i);
   }
 
-  dDebug("dnode mpeer is initialized, workers:%d qset:%p", tsMPeerPool.maxNum, tsMPeerQset);
+  dDebug("dnode mpeer is initialized, workers:%d qset:%p", tsMPeerWP.maxNum, tsMPeerQset);
   return 0;
 }
 
-void dnodeCleanupMnodePeer() {
-  for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) {
-    SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
+void dnodeCleanupMPeer() {
+  for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) {
+    SMPeerWorker *pWorker = tsMPeerWP.worker + i;
     if (pWorker->thread) {
       taosQsetThreadResume(tsMPeerQset);
     }
     dDebug("dnode mpeer worker:%d is closed", i);
   }
 
-  for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) {
-    SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
+  for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) {
+    SMPeerWorker *pWorker = tsMPeerWP.worker + i;
     dDebug("dnode mpeer worker:%d start to join", i);
     if (pWorker->thread) {
       pthread_join(pWorker->thread, NULL);
@@ -84,61 +84,60 @@ void dnodeCleanupMnodePeer() {
 
   taosCloseQset(tsMPeerQset);
   tsMPeerQset = NULL;
-  taosTFree(tsMPeerPool.peerWorker);
+  taosTFree(tsMPeerWP.worker);
 }
 
-int32_t dnodeAllocateMnodePqueue() {
+int32_t dnodeAllocateMPeerQueue() {
   tsMPeerQueue = taosOpenQueue();
   if (tsMPeerQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;
 
   taosAddIntoQset(tsMPeerQset, tsMPeerQueue, NULL);
 
-  for (int32_t i = tsMPeerPool.curNum; i < tsMPeerPool.maxNum; ++i) {
-    SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i;
+  for (int32_t i = tsMPeerWP.curNum; i < tsMPeerWP.maxNum; ++i) {
+    SMPeerWorker *pWorker = tsMPeerWP.worker + i;
     pWorker->workerId = i;
 
     pthread_attr_t thAttr;
     pthread_attr_init(&thAttr);
     pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
 
-    if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodePeerQueue, pWorker) != 0) {
+    if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMPeerQueue, pWorker) != 0) {
       dError("failed to create thread to process mpeer queue, reason:%s", strerror(errno));
     }
 
     pthread_attr_destroy(&thAttr);
 
-    tsMPeerPool.curNum = i + 1;
-    dDebug("dnode mpeer worker:%d is launched, total:%d", pWorker->workerId, tsMPeerPool.maxNum);
+    tsMPeerWP.curNum = i + 1;
+    dDebug("dnode mpeer worker:%d is launched, total:%d", pWorker->workerId, tsMPeerWP.maxNum);
   }
 
   dDebug("dnode mpeer queue:%p is allocated", tsMPeerQueue);
   return TSDB_CODE_SUCCESS;
 }
 
-void dnodeFreeMnodePqueue() {
+void dnodeFreeMPeerQueue() {
   dDebug("dnode mpeer queue:%p is freed", tsMPeerQueue);
   taosCloseQueue(tsMPeerQueue);
   tsMPeerQueue = NULL;
 }
 
-void dnodeDispatchToMnodePeerQueue(SRpcMsg *pMsg) {
+void dnodeDispatchToMPeerQueue(SRpcMsg *pMsg) {
   if (!mnodeIsRunning() || tsMPeerQueue == NULL) {
     dnodeSendRedirectMsg(pMsg, false);
-    rpcFreeCont(pMsg->pCont);
-    return;
+  } else {
+    SMnodeMsg *pPeer = mnodeCreateMsg(pMsg);
+    taosWriteQitem(tsMPeerQueue, TAOS_QTYPE_RPC, pPeer);
   }
 
-  SMnodeMsg *pPeer = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg));
-  mnodeCreateMsg(pPeer, pMsg);
-  taosWriteQitem(tsMPeerQueue, TAOS_QTYPE_RPC, pPeer);
+  rpcFreeCont(pMsg->pCont);
 }
 
-static void dnodeFreeMnodePeerMsg(SMnodeMsg *pPeer) {
+static void dnodeFreeMPeerMsg(SMnodeMsg *pPeer) {
   mnodeCleanupMsg(pPeer);
   taosFreeQitem(pPeer);
 }
 
-static void dnodeSendRpcMnodePeerRsp(SMnodeMsg *pPeer, int32_t code) {
+static void dnodeSendRpcMPeerRsp(SMnodeMsg *pPeer, int32_t code) {
   if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return;
 
   SRpcMsg rpcRsp = {
@@ -149,10 +148,10 @@ static void dnodeSendRpcMnodePeerRsp(SMnodeMsg *pPeer, int32_t code) {
   };
 
   rpcSendResponse(&rpcRsp);
-  dnodeFreeMnodePeerMsg(pPeer);
+  dnodeFreeMPeerMsg(pPeer);
 }
 
-static void *dnodeProcessMnodePeerQueue(void *param) {
+static void *dnodeProcessMPeerQueue(void *param) {
   SMnodeMsg *pPeerMsg;
   int32_t    type;
   void *     unUsed;
@@ -165,7 +164,7 @@ static void *dnodeProcessMnodePeerQueue(void *param) {
 
     dDebug("msg:%s will be processed in mpeer queue", taosMsg[pPeerMsg->rpcMsg.msgType]);    
     int32_t code = mnodeProcessPeerReq(pPeerMsg);    
-    dnodeSendRpcMnodePeerRsp(pPeerMsg, code);    
+    dnodeSendRpcMPeerRsp(pPeerMsg, code);    
   }
 
   return NULL;

src/dnode/src/dnodeMRead.c

@@ -35,46 +35,46 @@ typedef struct {
 typedef struct {
   int32_t curNum;
   int32_t maxNum;
-  SMReadWorker *readWorker;
+  SMReadWorker *worker;
 } SMReadWorkerPool;
 
-static SMReadWorkerPool tsMReadPool;
+static SMReadWorkerPool tsMReadWP;
 static taos_qset        tsMReadQset;
 static taos_queue       tsMReadQueue;
 
-static void *dnodeProcessMnodeReadQueue(void *param);
+static void *dnodeProcessMReadQueue(void *param);
 
-int32_t dnodeInitMnodeRead() {
+int32_t dnodeInitMRead() {
   tsMReadQset = taosOpenQset();
 
-  tsMReadPool.maxNum = tsNumOfCores * tsNumOfThreadsPerCore / 2;
-  tsMReadPool.maxNum = MAX(2, tsMReadPool.maxNum);
-  tsMReadPool.maxNum = MIN(4, tsMReadPool.maxNum);
-  tsMReadPool.curNum = 0;
-  tsMReadPool.readWorker = (SMReadWorker *)calloc(sizeof(SMReadWorker), tsMReadPool.maxNum);
+  tsMReadWP.maxNum = tsNumOfCores * tsNumOfThreadsPerCore / 2;
+  tsMReadWP.maxNum = MAX(2, tsMReadWP.maxNum);
+  tsMReadWP.maxNum = MIN(4, tsMReadWP.maxNum);
+  tsMReadWP.curNum = 0;
+  tsMReadWP.worker = (SMReadWorker *)calloc(sizeof(SMReadWorker), tsMReadWP.maxNum);
 
-  if (tsMReadPool.readWorker == NULL) return -1;
-  for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) {
-    SMReadWorker *pWorker = tsMReadPool.readWorker + i;
+  if (tsMReadWP.worker == NULL) return -1;
+  for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) {
+    SMReadWorker *pWorker = tsMReadWP.worker + i;
     pWorker->workerId = i;
     dDebug("dnode mread worker:%d is created", i);
   }
 
-  dDebug("dnode mread is initialized, workers:%d qset:%p", tsMReadPool.maxNum, tsMReadQset);
+  dDebug("dnode mread is initialized, workers:%d qset:%p", tsMReadWP.maxNum, tsMReadQset);
   return 0;
 }
 
-void dnodeCleanupMnodeRead() {
-  for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) {
-    SMReadWorker *pWorker = tsMReadPool.readWorker + i;
+void dnodeCleanupMRead() {
+  for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) {
+    SMReadWorker *pWorker = tsMReadWP.worker + i;
     if (pWorker->thread) {
       taosQsetThreadResume(tsMReadQset);
     }
     dDebug("dnode mread worker:%d is closed", i);
   }
 
-  for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) {
-    SMReadWorker *pWorker = tsMReadPool.readWorker + i;
+  for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) {
+    SMReadWorker *pWorker = tsMReadWP.worker + i;
     dDebug("dnode mread worker:%d start to join", i);
     if (pWorker->thread) {
       pthread_join(pWorker->thread, NULL);
@@ -86,64 +86,63 @@ void dnodeCleanupMnodeRead() {
 
   taosCloseQset(tsMReadQset);
   tsMReadQset = NULL;
-  free(tsMReadPool.readWorker);
+  free(tsMReadWP.worker);
 }
 
-int32_t dnodeAllocateMnodeRqueue() {
+int32_t dnodeAllocMReadQueue() {
   tsMReadQueue = taosOpenQueue();
   if (tsMReadQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;
 
   taosAddIntoQset(tsMReadQset, tsMReadQueue, NULL);
 
-  for (int32_t i = tsMReadPool.curNum; i < tsMReadPool.maxNum; ++i) {
-    SMReadWorker *pWorker = tsMReadPool.readWorker + i;
+  for (int32_t i = tsMReadWP.curNum; i < tsMReadWP.maxNum; ++i) {
+    SMReadWorker *pWorker = tsMReadWP.worker + i;
     pWorker->workerId = i;
 
     pthread_attr_t thAttr;
     pthread_attr_init(&thAttr);
     pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
 
-    if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodeReadQueue, pWorker) != 0) {
+    if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMReadQueue, pWorker) != 0) {
       dError("failed to create thread to process mread queue, reason:%s", strerror(errno));
     }
 
     pthread_attr_destroy(&thAttr);
-    tsMReadPool.curNum = i + 1;
-    dDebug("dnode mread worker:%d is launched, total:%d", pWorker->workerId, tsMReadPool.maxNum);
+    tsMReadWP.curNum = i + 1;
+    dDebug("dnode mread worker:%d is launched, total:%d", pWorker->workerId, tsMReadWP.maxNum);
   }
 
   dDebug("dnode mread queue:%p is allocated", tsMReadQueue);
   return TSDB_CODE_SUCCESS;
 }
 
-void dnodeFreeMnodeRqueue() {
+void dnodeFreeMReadQueue() {
   dDebug("dnode mread queue:%p is freed", tsMReadQueue);
   taosCloseQueue(tsMReadQueue);
   tsMReadQueue = NULL;
 }
 
-void dnodeDispatchToMnodeReadQueue(SRpcMsg *pMsg) {
+void dnodeDispatchToMReadQueue(SRpcMsg *pMsg) {
   if (!mnodeIsRunning() || tsMReadQueue == NULL) {
     dnodeSendRedirectMsg(pMsg, true);
-    rpcFreeCont(pMsg->pCont);
-    return;
+  } else {
+    SMnodeMsg *pRead = mnodeCreateMsg(pMsg);
+    taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead);
   }
 
-  SMnodeMsg *pRead = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg));
-  mnodeCreateMsg(pRead, pMsg);
-  taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead);
+  rpcFreeCont(pMsg->pCont);
 }
 
-static void dnodeFreeMnodeReadMsg(SMnodeMsg *pRead) {
+static void dnodeFreeMReadMsg(SMnodeMsg *pRead) {
   mnodeCleanupMsg(pRead);
   taosFreeQitem(pRead);
 }
 
-static void dnodeSendRpcMnodeReadRsp(SMnodeMsg *pRead, int32_t code) {
+static void dnodeSendRpcMReadRsp(SMnodeMsg *pRead, int32_t code) {
   if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return;
   if (code == TSDB_CODE_MND_ACTION_NEED_REPROCESSED) {
     // may be a auto create req, should put into write queue
-    dnodeReprocessMnodeWriteMsg(pRead);
+    dnodeReprocessMWriteMsg(pRead);
     return;
   }
 
@@ -155,23 +154,23 @@ static void dnodeSendRpcMnodeReadRsp(SMnodeMsg *pRead, int32_t code) {
   };
 
   rpcSendResponse(&rpcRsp);
-  dnodeFreeMnodeReadMsg(pRead);
+  dnodeFreeMReadMsg(pRead);
 }
 
-static void *dnodeProcessMnodeReadQueue(void *param) {
-  SMnodeMsg *pReadMsg;
+static void *dnodeProcessMReadQueue(void *param) {
+  SMnodeMsg *pRead;
   int32_t    type;
   void *     unUsed;
-  
+
   while (1) {
-    if (taosReadQitemFromQset(tsMReadQset, &type, (void **)&pReadMsg, &unUsed) == 0) {
+    if (taosReadQitemFromQset(tsMReadQset, &type, (void **)&pRead, &unUsed) == 0) {
       dDebug("qset:%p, mnode read got no message from qset, exiting", tsMReadQset);
       break;
     }
 
-    dDebug("%p, msg:%s will be processed in mread queue", pReadMsg->rpcMsg.ahandle, taosMsg[pReadMsg->rpcMsg.msgType]);    
-    int32_t code = mnodeProcessRead(pReadMsg);    
-    dnodeSendRpcMnodeReadRsp(pReadMsg, code);   
+    dDebug("%p, msg:%s will be processed in mread queue", pRead->rpcMsg.ahandle, taosMsg[pRead->rpcMsg.msgType]);
+    int32_t code = mnodeProcessRead(pRead);
+    dnodeSendRpcMReadRsp(pRead, code);
   }
 
   return NULL;

src/dnode/src/dnodeMWrite.c

@@ -36,45 +36,45 @@ typedef struct {
 typedef struct {
   int32_t curNum;
   int32_t maxNum;
-  SMWriteWorker *writeWorker;
+  SMWriteWorker *worker;
 } SMWriteWorkerPool;
 
-static SMWriteWorkerPool tsMWritePool;
+static SMWriteWorkerPool tsMWriteWP;
 static taos_qset         tsMWriteQset;
 static taos_queue        tsMWriteQueue;
 extern void *            tsDnodeTmr;
 
-static void *dnodeProcessMnodeWriteQueue(void *param);
+static void *dnodeProcessMWriteQueue(void *param);
 
-int32_t dnodeInitMnodeWrite() {
+int32_t dnodeInitMWrite() {
   tsMWriteQset = taosOpenQset();
 
-  tsMWritePool.maxNum = 1;
-  tsMWritePool.curNum = 0;
-  tsMWritePool.writeWorker = (SMWriteWorker *)calloc(sizeof(SMWriteWorker), tsMWritePool.maxNum);
+  tsMWriteWP.maxNum = 1;
+  tsMWriteWP.curNum = 0;
+  tsMWriteWP.worker = (SMWriteWorker *)calloc(sizeof(SMWriteWorker), tsMWriteWP.maxNum);
 
-  if (tsMWritePool.writeWorker == NULL) return -1;
-  for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) {
-    SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
+  if (tsMWriteWP.worker == NULL) return -1;
+  for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) {
+    SMWriteWorker *pWorker = tsMWriteWP.worker + i;
     pWorker->workerId = i;
     dDebug("dnode mwrite worker:%d is created", i);
   }
 
-  dDebug("dnode mwrite is initialized, workers:%d qset:%p", tsMWritePool.maxNum, tsMWriteQset);
+  dDebug("dnode mwrite is initialized, workers:%d qset:%p", tsMWriteWP.maxNum, tsMWriteQset);
   return 0;
 }
 
-void dnodeCleanupMnodeWrite() {
-  for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) {
-    SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
+void dnodeCleanupMWrite() {
+  for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) {
+    SMWriteWorker *pWorker = tsMWriteWP.worker + i;
     if (pWorker->thread) {
       taosQsetThreadResume(tsMWriteQset);
     }
     dDebug("dnode mwrite worker:%d is closed", i);
   }
 
-  for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) {
-    SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
+  for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) {
+    SMWriteWorker *pWorker = tsMWriteWP.worker + i;
     dDebug("dnode mwrite worker:%d start to join", i);
     if (pWorker->thread) {
       pthread_join(pWorker->thread, NULL);
@@ -86,58 +86,56 @@ void dnodeCleanupMnodeWrite() {
 
   taosCloseQset(tsMWriteQset);
   tsMWriteQset = NULL;
-  taosTFree(tsMWritePool.writeWorker);
+  taosTFree(tsMWriteWP.worker);
 }
 
-int32_t dnodeAllocateMnodeWqueue() {
+int32_t dnodeAllocMWritequeue() {
   tsMWriteQueue = taosOpenQueue();
   if (tsMWriteQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY;
 
   taosAddIntoQset(tsMWriteQset, tsMWriteQueue, NULL);
 
-  for (int32_t i = tsMWritePool.curNum; i < tsMWritePool.maxNum; ++i) {
-    SMWriteWorker *pWorker = tsMWritePool.writeWorker + i;
+  for (int32_t i = tsMWriteWP.curNum; i < tsMWriteWP.maxNum; ++i) {
+    SMWriteWorker *pWorker = tsMWriteWP.worker + i;
     pWorker->workerId = i;
 
     pthread_attr_t thAttr;
     pthread_attr_init(&thAttr);
     pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
 
-    if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodeWriteQueue, pWorker) != 0) {
+    if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMWriteQueue, pWorker) != 0) {
       dError("failed to create thread to process mwrite queue, reason:%s", strerror(errno));
     }
 
     pthread_attr_destroy(&thAttr);
-    tsMWritePool.curNum = i + 1;
-    dDebug("dnode mwrite worker:%d is launched, total:%d", pWorker->workerId, tsMWritePool.maxNum);
+    tsMWriteWP.curNum = i + 1;
+    dDebug("dnode mwrite worker:%d is launched, total:%d", pWorker->workerId, tsMWriteWP.maxNum);
   }
 
   dDebug("dnode mwrite queue:%p is allocated", tsMWriteQueue);
   return TSDB_CODE_SUCCESS;
 }
 
-void dnodeFreeMnodeWqueue() {
+void dnodeFreeMWritequeue() {
   dDebug("dnode mwrite queue:%p is freed", tsMWriteQueue);
   taosCloseQueue(tsMWriteQueue);
   tsMWriteQueue = NULL;
 }
 
-void dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg) {
+void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) {
   if (!mnodeIsRunning() || tsMWriteQueue == NULL) {
     dnodeSendRedirectMsg(pMsg, true);
-    rpcFreeCont(pMsg->pCont);
-    return;
+  } else {
+    SMnodeMsg *pWrite = mnodeCreateMsg(pMsg);
+    dDebug("app:%p:%p, msg:%s is put into mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite,
+           taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
+    taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);
   }
 
-  SMnodeMsg *pWrite = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg));
-  mnodeCreateMsg(pWrite, pMsg);
-
-  dDebug("app:%p:%p, msg:%s is put into mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite,
-         taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
-  taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite);
+  rpcFreeCont(pMsg->pCont);
 }
 
-static void dnodeFreeMnodeWriteMsg(SMnodeMsg *pWrite) {
+static void dnodeFreeMWriteMsg(SMnodeMsg *pWrite) {
   dDebug("app:%p:%p, msg:%s is freed from mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite,
          taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue);
 
@@ -145,12 +143,12 @@ static void dnodeFreeMnodeWriteMsg(SMnodeMsg *pWrite) {
   taosFreeQitem(pWrite);
 }
 
-void dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code) {
+void dnodeSendRpcMWriteRsp(void *pMsg, int32_t code) {
   SMnodeMsg *pWrite = pMsg;
   if (pWrite == NULL) return;
   if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return;
   if (code == TSDB_CODE_MND_ACTION_NEED_REPROCESSED) {
-    dnodeReprocessMnodeWriteMsg(pWrite);
+    dnodeReprocessMWriteMsg(pWrite);
     return;
   }
 
@@ -162,10 +160,10 @@ void dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code) {
   };
 
   rpcSendResponse(&rpcRsp);
-  dnodeFreeMnodeWriteMsg(pWrite);
+  dnodeFreeMWriteMsg(pWrite);
 }
 
-static void *dnodeProcessMnodeWriteQueue(void *param) {
+static void *dnodeProcessMWriteQueue(void *param) {
   SMnodeMsg *pWrite;
   int32_t    type;
   void *     unUsed;
@@ -180,13 +178,13 @@ static void *dnodeProcessMnodeWriteQueue(void *param) {
            taosMsg[pWrite->rpcMsg.msgType]);
 
     int32_t code = mnodeProcessWrite(pWrite);
-    dnodeSendRpcMnodeWriteRsp(pWrite, code);
+    dnodeSendRpcMWriteRsp(pWrite, code);
   }
 
   return NULL;
 }
 
-void dnodeReprocessMnodeWriteMsg(void *pMsg) {
+void dnodeReprocessMWriteMsg(void *pMsg) {
   SMnodeMsg *pWrite = pMsg;
 
   if (!mnodeIsRunning() || tsMWriteQueue == NULL) {
@@ -194,7 +192,7 @@ void dnodeReprocessMnodeWriteMsg(void *pMsg) {
            taosMsg[pWrite->rpcMsg.msgType], pWrite->retry);
 
     dnodeSendRedirectMsg(pMsg, true);
-    dnodeFreeMnodeWriteMsg(pWrite);
+    dnodeFreeMWriteMsg(pWrite);
   } else {
     dDebug("app:%p:%p, msg:%s is reput into mwrite queue:%p, retry times:%d", pWrite->rpcMsg.ahandle, pWrite,
            taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue, pWrite->retry);
@@ -203,12 +201,12 @@ void dnodeReprocessMnodeWriteMsg(void *pMsg) {
   }
 }
 
-static void dnodeDoDelayReprocessMnodeWriteMsg(void *param, void *tmrId) {
-  dnodeReprocessMnodeWriteMsg(param);
+static void dnodeDoDelayReprocessMWriteMsg(void *param, void *tmrId) {
+  dnodeReprocessMWriteMsg(param);
 }
 
-void dnodeDelayReprocessMnodeWriteMsg(void *pMsg) {
+void dnodeDelayReprocessMWriteMsg(void *pMsg) {
   SMnodeMsg *mnodeMsg = pMsg;
   void *unUsed = NULL;
-  taosTmrReset(dnodeDoDelayReprocessMnodeWriteMsg, 300, mnodeMsg, tsDnodeTmr, &unUsed);
+  taosTmrReset(dnodeDoDelayReprocessMWriteMsg, 300, mnodeMsg, tsDnodeTmr, &unUsed);
 }

src/dnode/src/dnodeMain.c

@@ -37,11 +37,11 @@
 #include "dnodeShell.h"
 #include "dnodeTelemetry.h"
 
-static SDnodeRunStatus tsDnodeRunStatus = TSDB_DNODE_RUN_STATUS_STOPPED;
+static SRunStatus tsRunStatus = TSDB_RUN_STATUS_STOPPED;
 
 static int32_t dnodeInitStorage();
 static void    dnodeCleanupStorage();
-static void    dnodeSetRunStatus(SDnodeRunStatus status);
+static void    dnodeSetRunStatus(SRunStatus status);
 static void    dnodeCheckDataDirOpenned(char *dir);
 static int32_t dnodeInitComponents();
 static void    dnodeCleanupComponents(int32_t stepId);
@@ -63,9 +63,9 @@ static const SDnodeComponent tsDnodeComponents[] = {
   {"check",     dnodeInitCheck,      dnodeCleanupCheck},     // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!!
   {"vread",     dnodeInitVRead,      dnodeCleanupVRead},
   {"vwrite",    dnodeInitVWrite,     dnodeCleanupVWrite},
-  {"mread",     dnodeInitMnodeRead,  dnodeCleanupMnodeRead},
-  {"mwrite",    dnodeInitMnodeWrite, dnodeCleanupMnodeWrite},
-  {"mpeer",     dnodeInitMnodePeer,  dnodeCleanupMnodePeer},  
+  {"mread",     dnodeInitMRead,      dnodeCleanupMRead},
+  {"mwrite",    dnodeInitMWrite,     dnodeCleanupMWrite},
+  {"mpeer",     dnodeInitMPeer,      dnodeCleanupMPeer},  
   {"client",    dnodeInitClient,     dnodeCleanupClient},
   {"server",    dnodeInitServer,     dnodeCleanupServer},
   {"mgmt",      dnodeInitMgmt,       dnodeCleanupMgmt},
@@ -104,7 +104,7 @@ static int32_t dnodeInitComponents() {
 }
 
 int32_t dnodeInitSystem() {
-  dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_INITIALIZE);
+  dnodeSetRunStatus(TSDB_RUN_STATUS_INITIALIZE);
   tscEmbedded  = 1;
   taosBlockSIGPIPE();
   taosResolveCRC();
@@ -137,7 +137,7 @@ int32_t dnodeInitSystem() {
   }
 
   dnodeStartModules();
-  dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_RUNING);
+  dnodeSetRunStatus(TSDB_RUN_STATUS_RUNING);
 
   dInfo("TDengine is initialized successfully");
 
@@ -145,20 +145,20 @@ int32_t dnodeInitSystem() {
 }
 
 void dnodeCleanUpSystem() {
-  if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_STOPPED) {
-    dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_STOPPED);
+  if (dnodeGetRunStatus() != TSDB_RUN_STATUS_STOPPED) {
+    dnodeSetRunStatus(TSDB_RUN_STATUS_STOPPED);
     dnodeCleanupComponents(sizeof(tsDnodeComponents) / sizeof(tsDnodeComponents[0]) - 1);
     taos_cleanup();
     taosCloseLog();
   }
 }
 
-SDnodeRunStatus dnodeGetRunStatus() {
-  return tsDnodeRunStatus;
+SRunStatus dnodeGetRunStatus() {
+  return tsRunStatus;
 }
 
-static void dnodeSetRunStatus(SDnodeRunStatus status) {
-  tsDnodeRunStatus = status;
+static void dnodeSetRunStatus(SRunStatus status) {
+  tsRunStatus = status;
 }
 
 static void dnodeCheckDataDirOpenned(char *dir) {

src/dnode/src/dnodeMgmt.c

@@ -47,6 +47,11 @@ typedef struct {
   int32_t * vnodeList;
 } SOpenVnodeThread;
 
+typedef struct {
+  SRpcMsg rpcMsg;
+  char    pCont[];
+} SMgmtMsg;
+
 void *            tsDnodeTmr = NULL;
 static void *     tsStatusTimer = NULL;
 static uint32_t   tsRebootTime;
@@ -172,38 +177,46 @@ void dnodeCleanupMgmt() {
   vnodeCleanupResources();
 }
 
-void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) {
-  void *item;
+static int32_t dnodeWriteToMgmtQueue(SRpcMsg *pMsg) {
+  int32_t   size = sizeof(SMgmtMsg) + pMsg->contLen;
+  SMgmtMsg *pMgmt = taosAllocateQitem(size);
+  if (pMgmt == NULL) {
+    return TSDB_CODE_DND_OUT_OF_MEMORY;
+  }
 
-  item = taosAllocateQitem(sizeof(SRpcMsg));
-  if (item) {
-    memcpy(item, pMsg, sizeof(SRpcMsg));
-    taosWriteQitem(tsMgmtQueue, 1, item);
-  } else {
-    SRpcMsg rsp = {
-      .handle = pMsg->handle,
-      .pCont  = NULL,
-      .code   = TSDB_CODE_DND_OUT_OF_MEMORY
-    };
-    
+  pMgmt->rpcMsg = *pMsg;
+  pMgmt->rpcMsg.pCont = pMgmt->pCont;
+  memcpy(pMgmt->pCont, pMsg->pCont, pMsg->contLen);
+  taosWriteQitem(tsMgmtQueue, TAOS_QTYPE_RPC, pMgmt);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) {
+  int32_t code = dnodeWriteToMgmtQueue(pMsg);
+  if (code != TSDB_CODE_SUCCESS) {
+    SRpcMsg rsp = {.handle = pMsg->handle, .code = code};
     rpcSendResponse(&rsp);
-    rpcFreeCont(pMsg->pCont);
   }
+
+  rpcFreeCont(pMsg->pCont);
 }
 
 static void *dnodeProcessMgmtQueue(void *param) {
-  SRpcMsg *pMsg;
-  SRpcMsg  rsp = {0};
-  int      type;
-  void *   handle;
+  SMgmtMsg *pMgmt;
+  SRpcMsg * pMsg;
+  SRpcMsg   rsp = {0};
+  int32_t   qtype;
+  void *    handle;
 
   while (1) {
-    if (taosReadQitemFromQset(tsMgmtQset, &type, (void **) &pMsg, &handle) == 0) {
+    if (taosReadQitemFromQset(tsMgmtQset, &qtype, (void **)&pMgmt, &handle) == 0) {
       dDebug("qset:%p, dnode mgmt got no message from qset, exit", tsMgmtQset);
       break;
     }
 
-    dDebug("%p, msg:%s will be processed", pMsg->ahandle, taosMsg[pMsg->msgType]);    
+    pMsg = &pMgmt->rpcMsg;
+    dDebug("%p, msg:%p:%s will be processed", pMsg->ahandle, pMgmt, taosMsg[pMsg->msgType]);
     if (dnodeProcessMgmtMsgFp[pMsg->msgType]) {
       rsp.code = (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg);
     } else {
@@ -211,10 +224,9 @@ static void *dnodeProcessMgmtQueue(void *param) {
     }
 
     rsp.handle = pMsg->handle;
-    rsp.pCont  = NULL;
+    rsp.pCont = NULL;
     rpcSendResponse(&rsp);
 
-    rpcFreeCont(pMsg->pCont);
     taosFreeQitem(pMsg);
   }
 

src/dnode/src/dnodePeer.c

@@ -19,6 +19,7 @@
  * to dnode. All theses messages are handled from here
  */
 
+#define _DEFAULT_SOURCE
 #include "os.h"
 #include "taosmsg.h"
 #include "tglobal.h"
@@ -34,8 +35,8 @@ static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
 static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *);
 static void (*dnodeProcessRspMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *rpcMsg);
 static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet);
-static void *tsDnodeServerRpc = NULL;
-static void *tsDnodeClientRpc = NULL;
+static void *tsServerRpc = NULL;
+static void *tsClientRpc = NULL;
 
 int32_t dnodeInitServer() {
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = dnodeDispatchToVWriteQueue;
@@ -50,11 +51,11 @@ int32_t dnodeInitServer() {
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToMgmtQueue;
   dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeDispatchToMgmtQueue;
 
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMnodePeerQueue;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMnodePeerQueue;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH]         = dnodeDispatchToMnodePeerQueue;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT]        = dnodeDispatchToMnodePeerQueue;
-  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS]       = dnodeDispatchToMnodePeerQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMPeerQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMPeerQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH]         = dnodeDispatchToMPeerQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT]        = dnodeDispatchToMPeerQueue;
+  dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS]       = dnodeDispatchToMPeerQueue;
   
   SRpcInit rpcInit;
   memset(&rpcInit, 0, sizeof(rpcInit));
@@ -66,8 +67,8 @@ int32_t dnodeInitServer() {
   rpcInit.connType     = TAOS_CONN_SERVER;
   rpcInit.idleTime     = tsShellActivityTimer * 1000;
 
-  tsDnodeServerRpc = rpcOpen(&rpcInit);
-  if (tsDnodeServerRpc == NULL) {
+  tsServerRpc = rpcOpen(&rpcInit);
+  if (tsServerRpc == NULL) {
     dError("failed to init inter-dnodes RPC server");
     return -1;
   }
@@ -77,9 +78,9 @@ int32_t dnodeInitServer() {
 }
 
 void dnodeCleanupServer() {
-  if (tsDnodeServerRpc) {
-    rpcClose(tsDnodeServerRpc);
-    tsDnodeServerRpc = NULL;
+  if (tsServerRpc) {
+    rpcClose(tsServerRpc);
+    tsServerRpc = NULL;
     dInfo("inter-dnodes RPC server is closed");
   }
 }
@@ -93,7 +94,7 @@ static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
   
   if (pMsg->pCont == NULL) return;
 
-  if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) {
+  if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) {
     rspMsg.code = TSDB_CODE_APP_NOT_READY;
     rpcSendResponse(&rspMsg);
     rpcFreeCont(pMsg->pCont);
@@ -131,8 +132,8 @@ int32_t dnodeInitClient() {
   rpcInit.ckey         = "key";
   rpcInit.secret       = secret;
 
-  tsDnodeClientRpc = rpcOpen(&rpcInit);
-  if (tsDnodeClientRpc == NULL) {
+  tsClientRpc = rpcOpen(&rpcInit);
+  if (tsClientRpc == NULL) {
     dError("failed to init mnode rpc client");
     return -1;
   }
@@ -142,9 +143,9 @@ int32_t dnodeInitClient() {
 }
 
 void dnodeCleanupClient() {
-  if (tsDnodeClientRpc) {
-    rpcClose(tsDnodeClientRpc);
-    tsDnodeClientRpc = NULL;
+  if (tsClientRpc) {
+    rpcClose(tsClientRpc);
+    tsClientRpc = NULL;
     dInfo("dnode inter-dnodes rpc client is closed");
   }
 }
@@ -168,15 +169,15 @@ void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) {
 }
 
 void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) {
-  rpcSendRequest(tsDnodeClientRpc, epSet, rpcMsg);
+  rpcSendRequest(tsClientRpc, epSet, rpcMsg);
 }
 
 void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) {
   SRpcEpSet epSet = {0};
   dnodeGetEpSetForPeer(&epSet);
-  rpcSendRecv(tsDnodeClientRpc, &epSet, rpcMsg, rpcRsp);
+  rpcSendRecv(tsClientRpc, &epSet, rpcMsg, rpcRsp);
 }
 
 void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet) {
-  rpcSendRecv(tsDnodeClientRpc, epSet, rpcMsg, rpcRsp);
+  rpcSendRecv(tsClientRpc, epSet, rpcMsg, rpcRsp);
 }
\ No newline at end of file

src/dnode/src/dnodeShell.c

@@ -33,9 +33,9 @@
 static void  (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *);
 static void    dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *);
 static int     dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey);
-static void  * tsDnodeShellRpc = NULL;
-static int32_t tsDnodeQueryReqNum  = 0;
-static int32_t tsDnodeSubmitReqNum = 0;
+static void  * tsShellRpc = NULL;
+static int32_t tsQueryReqNum  = 0;
+static int32_t tsSubmitReqNum = 0;
 
 int32_t dnodeInitShell() {
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_SUBMIT]         = dnodeDispatchToVWriteQueue;
@@ -44,35 +44,35 @@ int32_t dnodeInitShell() {
   dnodeProcessShellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeDispatchToVWriteQueue;
   
   // the following message shall be treated as mnode write
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT]  = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT]   = dnodeDispatchToMnodeWriteQueue; 
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMnodeWriteQueue; 
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER]  = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER]   = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE]  = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB]   = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB]     = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB]    = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE]  = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM]= dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY]  = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN]   = dnodeDispatchToMnodeWriteQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]= dnodeDispatchToMnodeWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT]  = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT]   = dnodeDispatchToMWriteQueue; 
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMWriteQueue; 
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER]  = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER]   = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE]  = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB]   = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB]     = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB]    = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE]  = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM]= dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY]  = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN]   = dnodeDispatchToMWriteQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]= dnodeDispatchToMWriteQueue;
   
   // the following message shall be treated as mnode query 
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT]   = dnodeDispatchToMnodeReadQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONNECT]     = dnodeDispatchToMnodeReadQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_USE_DB]      = dnodeDispatchToMnodeReadQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META]  = dnodeDispatchToMnodeReadQueue; 
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMnodeReadQueue;   
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMnodeReadQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW]        = dnodeDispatchToMnodeReadQueue;
-  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE]    = dnodeDispatchToMnodeReadQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT]   = dnodeDispatchToMReadQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONNECT]     = dnodeDispatchToMReadQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_USE_DB]      = dnodeDispatchToMReadQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META]  = dnodeDispatchToMReadQueue; 
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMReadQueue;   
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMReadQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW]        = dnodeDispatchToMReadQueue;
+  dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE]    = dnodeDispatchToMReadQueue;
 
   int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore;
   numOfThreads = (int32_t) ((1.0 - tsRatioOfQueryThreads) * numOfThreads / 2.0);
@@ -91,8 +91,8 @@ int32_t dnodeInitShell() {
   rpcInit.idleTime     = tsShellActivityTimer * 1000;
   rpcInit.afp          = dnodeRetrieveUserAuthInfo;
 
-  tsDnodeShellRpc = rpcOpen(&rpcInit);
-  if (tsDnodeShellRpc == NULL) {
+  tsShellRpc = rpcOpen(&rpcInit);
+  if (tsShellRpc == NULL) {
     dError("failed to init shell rpc server");
     return -1;
   }
@@ -102,13 +102,13 @@ int32_t dnodeInitShell() {
 }
 
 void dnodeCleanupShell() {
-  if (tsDnodeShellRpc) {
-    rpcClose(tsDnodeShellRpc);
-    tsDnodeShellRpc = NULL;
+  if (tsShellRpc) {
+    rpcClose(tsShellRpc);
+    tsShellRpc = NULL;
   }
 }
 
-void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
+static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
   SRpcMsg rpcMsg = {
     .handle  = pMsg->handle,
     .pCont   = NULL,
@@ -117,7 +117,7 @@ void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
 
   if (pMsg->pCont == NULL) return;
 
-  if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) {
+  if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) {
     dError("RPC %p, shell msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]);
     rpcMsg.code = TSDB_CODE_APP_NOT_READY;
     rpcSendResponse(&rpcMsg);
@@ -126,9 +126,9 @@ void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
   }
 
   if (pMsg->msgType == TSDB_MSG_TYPE_QUERY) {
-    atomic_fetch_add_32(&tsDnodeQueryReqNum, 1);
+    atomic_fetch_add_32(&tsQueryReqNum, 1);
   } else if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) {
-    atomic_fetch_add_32(&tsDnodeSubmitReqNum, 1);
+    atomic_fetch_add_32(&tsSubmitReqNum, 1);
   } else {}
 
   if ( dnodeProcessShellMsgFp[pMsg->msgType] ) {
@@ -211,12 +211,12 @@ void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid) {
   }
 }
 
-SDnodeStatisInfo dnodeGetStatisInfo() {
-  SDnodeStatisInfo info = {0};
-  if (dnodeGetRunStatus() == TSDB_DNODE_RUN_STATUS_RUNING) {
+SStatisInfo dnodeGetStatisInfo() {
+  SStatisInfo info = {0};
+  if (dnodeGetRunStatus() == TSDB_RUN_STATUS_RUNING) {
     info.httpReqNum   = httpGetReqCount();
-    info.queryReqNum  = atomic_exchange_32(&tsDnodeQueryReqNum, 0);
-    info.submitReqNum = atomic_exchange_32(&tsDnodeSubmitReqNum, 0);
+    info.queryReqNum  = atomic_exchange_32(&tsQueryReqNum, 0);
+    info.submitReqNum = atomic_exchange_32(&tsSubmitReqNum, 0);
   }
 
   return info;

src/dnode/src/dnodeVWrite.c

@@ -113,7 +113,7 @@ void dnodeDispatchToVWriteQueue(SRpcMsg *pRpcMsg) {
 void *dnodeAllocVWriteQueue(void *pVnode) {
   pthread_mutex_lock(&tsVWriteWP.mutex);
   SVWriteWorker *pWorker = tsVWriteWP.worker + tsVWriteWP.nextId;
-  void *queue = taosOpenQueue();
+  taos_queue *queue = taosOpenQueue();
   if (queue == NULL) {
     pthread_mutex_unlock(&tsVWriteWP.mutex);
     return NULL;

src/inc/dnode.h

@@ -27,16 +27,16 @@ typedef struct {
   int32_t queryReqNum;
   int32_t submitReqNum;
   int32_t httpReqNum;
-} SDnodeStatisInfo;
+} SStatisInfo;
 
 typedef enum {
-  TSDB_DNODE_RUN_STATUS_INITIALIZE,
-  TSDB_DNODE_RUN_STATUS_RUNING,
-  TSDB_DNODE_RUN_STATUS_STOPPED
-} SDnodeRunStatus;
+  TSDB_RUN_STATUS_INITIALIZE,
+  TSDB_RUN_STATUS_RUNING,
+  TSDB_RUN_STATUS_STOPPED
+} SRunStatus;
 
-SDnodeRunStatus dnodeGetRunStatus();
-SDnodeStatisInfo dnodeGetStatisInfo();
+SRunStatus  dnodeGetRunStatus();
+SStatisInfo dnodeGetStatisInfo();
 
 bool    dnodeIsFirstDeploy();
 bool    dnodeIsMasterEp(char *ep);
@@ -59,15 +59,15 @@ void  dnodeSendRpcVWriteRsp(void *pVnode, void *param, int32_t code);
 void *dnodeAllocVReadQueue(void *pVnode);
 void  dnodeFreeVReadQueue(void *rqueue);
 
-int32_t dnodeAllocateMnodePqueue();
-void    dnodeFreeMnodePqueue();
-int32_t dnodeAllocateMnodeRqueue();
-void    dnodeFreeMnodeRqueue();
-int32_t dnodeAllocateMnodeWqueue();
-void    dnodeFreeMnodeWqueue();
-void    dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code);
-void    dnodeReprocessMnodeWriteMsg(void *pMsg);
-void    dnodeDelayReprocessMnodeWriteMsg(void *pMsg);
+int32_t dnodeAllocateMPeerQueue();
+void    dnodeFreeMPeerQueue();
+int32_t dnodeAllocMReadQueue();
+void    dnodeFreeMReadQueue();
+int32_t dnodeAllocMWritequeue();
+void    dnodeFreeMWritequeue();
+void    dnodeSendRpcMWriteRsp(void *pMsg, int32_t code);
+void    dnodeReprocessMWriteMsg(void *pMsg);
+void    dnodeDelayReprocessMWriteMsg(void *pMsg);
 
 void    dnodeSendStatusMsgToMnode();
 

src/inc/mnode.h

@@ -35,7 +35,13 @@ typedef struct {
 } SMnodeRsp;
 
 typedef struct SMnodeMsg {
-  SRpcMsg   rpcMsg;
+  struct SAcctObj * pAcct;
+  struct SDnodeObj *pDnode;
+  struct SUserObj * pUser;
+  struct SDbObj *   pDb;
+  struct SVgObj *   pVgroup;
+  struct STableObj *pTable;
+  struct SSTableObj*pSTable;
   SMnodeRsp rpcRsp;
   int8_t    received;
   int8_t    successed;
@@ -43,16 +49,11 @@ typedef struct SMnodeMsg {
   int8_t    retry;
   int32_t   code;
   void *    pObj;
-  struct SAcctObj * pAcct;
-  struct SDnodeObj *pDnode;
-  struct SUserObj * pUser;
-  struct SDbObj *   pDb;
-  struct SVgObj *   pVgroup;
-  struct STableObj *pTable;
-  struct SSuperTableObj *pSTable;
+  SRpcMsg   rpcMsg;
+  char      pCont[];
 } SMnodeMsg;
 
-void    mnodeCreateMsg(SMnodeMsg *pMsg, SRpcMsg *rpcMsg);
+void *  mnodeCreateMsg(SRpcMsg *pRpcMsg);
 int32_t mnodeInitMsg(SMnodeMsg *pMsg);
 void    mnodeCleanupMsg(SMnodeMsg *pMsg);
 

src/inc/taosdef.h

@@ -429,42 +429,43 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
 #define TSDB_PORT_DNODEDNODE 5 
 #define TSDB_PORT_SYNC       10 
 #define TSDB_PORT_HTTP       11 
-#define TSDB_PORT_ARBITRATOR 12 
+#define TSDB_PORT_ARBITRATOR 12
 
-#define TAOS_QTYPE_RPC      0
-#define TAOS_QTYPE_FWD      1
-#define TAOS_QTYPE_WAL      2 
-#define TAOS_QTYPE_CQ       3
-#define TAOS_QTYPE_QUERY    4
+typedef enum {
+  TAOS_QTYPE_RPC   = 0,
+  TAOS_QTYPE_FWD   = 1,
+  TAOS_QTYPE_WAL   = 2,
+  TAOS_QTYPE_CQ    = 3,
+  TAOS_QTYPE_QUERY = 4
+} EQType;
 
 typedef enum {
-  TSDB_SUPER_TABLE        = 0,  // super table
-  TSDB_CHILD_TABLE        = 1,  // table created from super table
-  TSDB_NORMAL_TABLE       = 2,  // ordinary table
-  TSDB_STREAM_TABLE       = 3,  // table created from stream computing
-  TSDB_TABLE_MAX          = 4
+  TSDB_SUPER_TABLE  = 0,   // super table
+  TSDB_CHILD_TABLE  = 1,   // table created from super table
+  TSDB_NORMAL_TABLE = 2,  // ordinary table
+  TSDB_STREAM_TABLE = 3,  // table created from stream computing
+  TSDB_TABLE_MAX    = 4
 } ETableType;
 
 typedef enum {
-  TSDB_MOD_MNODE,
-  TSDB_MOD_HTTP,
-  TSDB_MOD_MONITOR,
-  TSDB_MOD_MQTT,
-  TSDB_MOD_MAX
+  TSDB_MOD_MNODE   = 0,
+  TSDB_MOD_HTTP    = 1,
+  TSDB_MOD_MONITOR = 2,
+  TSDB_MOD_MQTT    = 3,
+  TSDB_MOD_MAX     = 4
 } EModuleType;
 
-  typedef enum {
-    TSDB_CHECK_ITEM_NETWORK,
-    TSDB_CHECK_ITEM_MEM,
-    TSDB_CHECK_ITEM_CPU,
-    TSDB_CHECK_ITEM_DISK,
-    TSDB_CHECK_ITEM_OS,    
-    TSDB_CHECK_ITEM_ACCESS,    
-    TSDB_CHECK_ITEM_VERSION,
-    TSDB_CHECK_ITEM_DATAFILE,
-    TSDB_CHECK_ITEM_MAX
-  } ECheckItemType;
-
+typedef enum {
+  TSDB_CHECK_ITEM_NETWORK,
+  TSDB_CHECK_ITEM_MEM,
+  TSDB_CHECK_ITEM_CPU,
+  TSDB_CHECK_ITEM_DISK,
+  TSDB_CHECK_ITEM_OS,    
+  TSDB_CHECK_ITEM_ACCESS,    
+  TSDB_CHECK_ITEM_VERSION,
+  TSDB_CHECK_ITEM_DATAFILE,
+  TSDB_CHECK_ITEM_MAX
+} ECheckItemType;
 
 #ifdef __cplusplus
 }

src/inc/tcq.h

@@ -24,7 +24,7 @@ extern "C" {
 typedef int32_t (*FCqWrite)(void *ahandle, void *pHead, int32_t qtype, void *pMsg);
 
 typedef struct {
-  int      vgId;
+  int32_t  vgId;
   char     user[TSDB_USER_LEN];
   char     pass[TSDB_PASSWORD_LEN];
   char     db[TSDB_DB_NAME_LEN];
@@ -42,12 +42,12 @@ void  cqStart(void *handle);
 void  cqStop(void *handle);
 
 // cqCreate is called by TSDB to start an instance of CQ 
-void *cqCreate(void *handle, uint64_t uid, int sid, char *sqlStr, STSchema *pSchema);
+void *cqCreate(void *handle, uint64_t uid, int32_t sid, char *sqlStr, STSchema *pSchema);
 
 // cqDrop is called by TSDB to stop an instance of CQ, handle is the return value of cqCreate
 void  cqDrop(void *handle);
 
-extern int cqDebugFlag;
+extern int32_t cqDebugFlag;
 
 
 #ifdef __cplusplus

src/inc/trpc.h

@@ -83,13 +83,13 @@ void  rpcClose(void *);
 void *rpcMallocCont(int contLen);
 void  rpcFreeCont(void *pCont);
 void *rpcReallocCont(void *ptr, int contLen);
-void  rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg);
+int64_t rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg);
 void  rpcSendResponse(const SRpcMsg *pMsg);
 void  rpcSendRedirectRsp(void *pConn, const SRpcEpSet *pEpSet); 
 int   rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
 void  rpcSendRecv(void *shandle, SRpcEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp);
 int   rpcReportProgress(void *pConn, char *pCont, int contLen);
-void  rpcCancelRequest(void *pContext);
+void  rpcCancelRequest(int64_t rid);
 
 #ifdef __cplusplus
 }

src/inc/tsdb.h

@@ -164,6 +164,12 @@ typedef struct STsdbQueryCond {
   SColumnInfo *colList;
 } STsdbQueryCond;
 
+typedef struct SMemRef {
+  int32_t  ref;
+  void    *mem;
+  void    *imem;
+} SMemRef;
+
 typedef struct SDataBlockInfo {
   STimeWindow window;
   int32_t     rows;
@@ -193,7 +199,7 @@ typedef struct {
  * @param qinfo      query info handle from query processor
  * @return
  */
-TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo);
+TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo, SMemRef* pRef);
 
 /**
  * Get the last row of the given query time window for all the tables in STableGroupInfo object.
@@ -205,7 +211,7 @@ TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab
  * @param tableInfo  table list.
  * @return
  */
-TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo);
+TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo, SMemRef* pRef);
 
 /**
  * get the queried table object list
@@ -223,7 +229,7 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle);
  * @return
  */
 TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList,
-                                               void *qinfo);
+                                               void *qinfo, SMemRef* pRef);
 
 /**
  * move to next block if exists

src/inc/tsync.h

@@ -106,13 +106,13 @@ typedef void* tsync_h;
 int32_t syncInit();
 void    syncCleanUp();
 
-tsync_h syncStart(const SSyncInfo *);
-void    syncStop(tsync_h shandle);
-int32_t syncReconfig(tsync_h shandle, const SSyncCfg *);
-int32_t syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype);
-void    syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code);
-void    syncRecover(tsync_h shandle);      // recover from other nodes:
-int     syncGetNodesRole(tsync_h shandle, SNodesRole *);
+int64_t syncStart(const SSyncInfo *);
+void    syncStop(int64_t rid);
+int32_t syncReconfig(int64_t rid, const SSyncCfg *);
+int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int qtype);
+void    syncConfirmForward(int64_t rid, uint64_t version, int32_t code);
+void    syncRecover(int64_t rid);      // recover from other nodes:
+int     syncGetNodesRole(int64_t rid, SNodesRole *);
 
 extern  char *syncRole[];
 

src/inc/twal.h

@@ -25,9 +25,15 @@ typedef enum {
   TAOS_WAL_FSYNC = 2
 } EWalType;
 
+typedef enum {
+  TAOS_WAL_NOT_KEEP = 0,
+  TAOS_WAL_KEEP = 1
+} EWalKeep;
+
 typedef struct {
   int8_t   msgType;
-  int8_t   reserved[3];
+  int8_t   sver;
+  int8_t   reserved[2];
   int32_t  len;
   uint64_t version;
   uint32_t signature;
@@ -36,11 +42,10 @@ typedef struct {
 } SWalHead;
 
 typedef struct {
-  int32_t vgId;
-  int32_t fsyncPeriod;  // millisecond
-  int8_t  walLevel;     // wal level
-  int8_t  wals;         // number of WAL files;
-  int8_t  keep;         // keep the wal file when closed
+  int32_t  vgId;
+  int32_t  fsyncPeriod;  // millisecond
+  EWalType walLevel;     // wal level
+  EWalKeep keep;         // keep the wal file when closed
 } SWalCfg;
 
 typedef void *  twalh;  // WAL HANDLE
@@ -58,7 +63,7 @@ int32_t walWrite(twalh, SWalHead *);
 void    walFsync(twalh, bool forceFsync);
 int32_t walRestore(twalh, void *pVnode, FWalWrite writeFp);
 int32_t walGetWalFile(twalh, char *fileName, int64_t *fileId);
-int64_t walGetVersion(twalh);
+uint64_t walGetVersion(twalh);
 
 #ifdef __cplusplus
 }

src/kit/taosdemo/taosdemo.c

@@ -793,9 +793,9 @@ int main(int argc, char *argv[]) {
           (ntables * nrecords_per_table) / (t * nrecords_per_request),
           t * 1000);
 
-  printf("Spent %.4f seconds to insert %d records with %d record(s) per request: %.2f records/second\n",
-         t, ntables * nrecords_per_table, nrecords_per_request,
-         ntables * nrecords_per_table / t);
+  printf("Spent %.4f seconds to insert %lld records with %d record(s) per request: %.2f records/second\n",
+         t, (long long int)ntables * nrecords_per_table, nrecords_per_request,
+         ((long long int)ntables * nrecords_per_table) / t);
 
   for (int i = 0; i < threads; i++) {
     info *t_info = infos + i;

src/mnode/inc/mnodeDef.h

@@ -89,7 +89,7 @@ typedef struct STableObj {
   int8_t type;
 } STableObj;
 
-typedef struct SSuperTableObj {
+typedef struct SSTableObj {
   STableObj  info; 
   int8_t     reserved0[9]; // for fill struct STableObj to 4byte align
   int16_t    nextColId;
@@ -104,7 +104,7 @@ typedef struct SSuperTableObj {
   int32_t    numOfTables;
   SSchema *  schema;
   void *     vgHash;
-} SSuperTableObj;
+} SSTableObj;
 
 typedef struct {
   STableObj  info;  
@@ -122,8 +122,8 @@ typedef struct {
   int32_t    refCount;
   char*      sql;          //used by normal table
   SSchema*   schema;       //used by normal table
-  SSuperTableObj *superTable;
-} SChildTableObj;
+  SSTableObj*superTable;
+} SCTableObj;
 
 typedef struct {
   int32_t    dnodeId;

src/mnode/inc/mnodeTable.h

@@ -29,8 +29,8 @@ int64_t mnodeGetChildTableNum();
 void *  mnodeGetTable(char *tableId);
 void    mnodeIncTableRef(void *pTable);
 void    mnodeDecTableRef(void *pTable);
-void *  mnodeGetNextChildTable(void *pIter, SChildTableObj **pTable);
-void *  mnodeGetNextSuperTable(void *pIter, SSuperTableObj **pTable);
+void *  mnodeGetNextChildTable(void *pIter, SCTableObj **pTable);
+void *  mnodeGetNextSuperTable(void *pIter, SSTableObj **pTable);
 void    mnodeDropAllChildTables(SDbObj *pDropDb);
 void    mnodeDropAllSuperTables(SDbObj *pDropDb);
 void    mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup);

src/mnode/inc/mnodeVgroup.h

@@ -43,8 +43,8 @@ void    mnodeDropVgroup(SVgObj *pVgroup, void *ahandle);
 void    mnodeAlterVgroup(SVgObj *pVgroup, void *ahandle);
 int32_t mnodeGetAvailableVgroup(struct SMnodeMsg *pMsg, SVgObj **pVgroup, int32_t *sid);
 
-void    mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable);
-void    mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable);
+void    mnodeAddTableIntoVgroup(SVgObj *pVgroup, SCTableObj *pTable);
+void    mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SCTableObj *pTable);
 void    mnodeSendDropVnodeMsg(int32_t vgId, SRpcEpSet *epSet, void *ahandle);
 void    mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle);
 void    mnodeSendAlterVgroupMsg(SVgObj *pVgroup);

src/mnode/src/mnodeInt.c

@@ -18,7 +18,7 @@
 #include "taosmsg.h"
 #include "taoserror.h"
 #include "trpc.h"
-#include "tcache.h"
+#include "tqueue.h"
 #include "mnode.h"
 #include "dnode.h"
 #include "mnodeDef.h"
@@ -34,8 +34,15 @@
 #include "mnodeUser.h"
 #include "mnodeVgroup.h"
 
-void mnodeCreateMsg(SMnodeMsg *pMsg, SRpcMsg *rpcMsg) {
-  pMsg->rpcMsg = *rpcMsg;
+void *mnodeCreateMsg(SRpcMsg *pRpcMsg) {
+  int32_t    size = sizeof(SMnodeMsg) + pRpcMsg->contLen;
+  SMnodeMsg *pMsg = taosAllocateQitem(size);
+
+  pMsg->rpcMsg = *pRpcMsg;
+  pMsg->rpcMsg.pCont = pMsg->pCont;
+  memcpy(pMsg->pCont, pRpcMsg->pCont, pRpcMsg->contLen);
+
+  return pMsg;
 }
 
 int32_t mnodeInitMsg(SMnodeMsg *pMsg) {
@@ -54,7 +61,9 @@ int32_t mnodeInitMsg(SMnodeMsg *pMsg) {
 
 void mnodeCleanupMsg(SMnodeMsg *pMsg) {
   if (pMsg != NULL) {
-    if (pMsg->rpcMsg.pCont) rpcFreeCont(pMsg->rpcMsg.pCont);
+    if (pMsg->rpcMsg.pCont != pMsg->pCont) {
+      tfree(pMsg->rpcMsg.pCont);
+    }
     if (pMsg->pUser) mnodeDecUserRef(pMsg->pUser);
     if (pMsg->pDb) mnodeDecDbRef(pMsg->pDb);
     if (pMsg->pVgroup) mnodeDecVgroupRef(pMsg->pVgroup);

src/mnode/src/mnodeMain.c

@@ -96,9 +96,9 @@ int32_t mnodeStartSystem() {
     return -1;
   }
 
-  dnodeAllocateMnodeWqueue();
-  dnodeAllocateMnodeRqueue();
-  dnodeAllocateMnodePqueue();
+  dnodeAllocMWritequeue();
+  dnodeAllocMReadQueue();
+  dnodeAllocateMPeerQueue();
 
   if (mnodeInitComponents() != 0) {
     return -1;
@@ -127,9 +127,9 @@ void mnodeCleanupSystem() {
     mInfo("starting to clean up mnode");
     tsMgmtIsRunning = false;
 
-    dnodeFreeMnodeWqueue();
-    dnodeFreeMnodeRqueue();
-    dnodeFreeMnodePqueue();
+    dnodeFreeMWritequeue();
+    dnodeFreeMReadQueue();
+    dnodeFreeMPeerQueue();
     mnodeCleanupTimer();
     mnodeCleanupComponents(sizeof(tsMnodeComponents) / sizeof(tsMnodeComponents[0]) - 1);
 

src/mnode/src/mnodeSdb.c

@@ -72,7 +72,7 @@ typedef struct {
   ESyncRole  role;
   ESdbStatus status;
   int64_t    version;
-  void *     sync;
+  int64_t    sync;
   void *     wal;
   SSyncCfg   cfg;
   int32_t    numOfTables;
@@ -175,7 +175,7 @@ static void *sdbGetTableFromId(int32_t tableId) {
 }
 
 static int32_t sdbInitWal() {
-  SWalCfg walCfg = {.vgId = 1, .walLevel = 2, .wals = 2, .keep = 1, .fsyncPeriod = 0};
+  SWalCfg walCfg = {.vgId = 1, .walLevel = TAOS_WAL_FSYNC, .keep = TAOS_WAL_KEEP, .fsyncPeriod = 0};
   char temp[TSDB_FILENAME_LEN];
   sprintf(temp, "%s/wal", tsMnodeDir);
   tsSdbObj.wal = walOpen(temp, &walCfg);
@@ -212,7 +212,7 @@ static void sdbRestoreTables() {
 }
 
 void sdbUpdateMnodeRoles() {
-  if (tsSdbObj.sync == NULL) return;
+  if (tsSdbObj.sync <= 0) return;
 
   SNodesRole roles = {0};
   syncGetNodesRole(tsSdbObj.sync, &roles);
@@ -295,7 +295,7 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) {
   if (pOper->writeCb != NULL) {
     pOper->retCode = (*pOper->writeCb)(pMsg, pOper->retCode);
   }
-  dnodeSendRpcMnodeWriteRsp(pMsg, pOper->retCode);
+  dnodeSendRpcMWriteRsp(pMsg, pOper->retCode);
 
   // if ahandle, means this func is called by sdb write
   if (ahandle == NULL) {
@@ -433,7 +433,7 @@ void sdbCleanUp() {
 
   if (tsSdbObj.sync) {
     syncStop(tsSdbObj.sync);
-    tsSdbObj.sync = NULL;
+    tsSdbObj.sync = -1;
   }
 
   if (tsSdbObj.wal) {
@@ -1043,7 +1043,7 @@ void sdbFreeWritequeue() {
 int32_t sdbWriteToQueue(void *param, void *data, int32_t qtype, void *pMsg) {
   SWalHead *pHead = data;
   int32_t   size = sizeof(SWalHead) + pHead->len;
-  SWalHead *pWal = (SWalHead *)taosAllocateQitem(size);
+  SWalHead *pWal = taosAllocateQitem(size);
   memcpy(pWal, pHead, size);
 
   taosWriteQitem(tsSdbWriteQueue, qtype, pWal);

src/mnode/src/mnodeVgroup.c

@@ -529,7 +529,7 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) {
     SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb};
     (void)sdbUpdateRow(&desc);
 
-    dnodeReprocessMnodeWriteMsg(pMsg);
+    dnodeReprocessMWriteMsg(pMsg);
     return TSDB_CODE_MND_ACTION_IN_PROGRESS;
     // if (pVgroup->status == TAOS_VG_STATUS_CREATING || pVgroup->status == TAOS_VG_STATUS_READY) {
     //   mInfo("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId,
@@ -537,7 +537,7 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) {
     //   pVgroup->status = TAOS_VG_STATUS_READY;
     //   SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb};
     //   (void)sdbUpdateRow(&desc);
-    //   dnodeReprocessMnodeWriteMsg(pMsg);
+    //   dnodeReprocessMWriteMsg(pMsg);
     //   return TSDB_CODE_MND_ACTION_IN_PROGRESS;
     // } else {
     //   mError("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d, but vgroup is dropping", pMsg->rpcMsg.ahandle,
@@ -694,7 +694,7 @@ static bool mnodeFilterVgroups(SVgObj *pVgroup, STableObj *pTable) {
     return true;
   }
 
-  SChildTableObj *pCTable = (SChildTableObj *)pTable;
+  SCTableObj *pCTable = (SCTableObj *)pTable;
   if (pVgroup->vgId == pCTable->vgId) {
     return true;
   } else {
@@ -791,7 +791,7 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v
   return numOfRows;
 }
 
-void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
+void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SCTableObj *pTable) {
   int32_t idPoolSize = taosIdPoolMaxSize(pVgroup->idPool);
   if (pTable->tid > idPoolSize) {
     mnodeAllocVgroupIdPool(pVgroup);
@@ -807,7 +807,7 @@ void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
   }
 }
 
-void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
+void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SCTableObj *pTable) {
   if (pTable->tid >= 1) {
     taosFreeId(pVgroup->idPool, pTable->tid);
     pVgroup->numOfTables--;
@@ -970,7 +970,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
     if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
       mnodeMsg->pVgroup = NULL;
       mnodeDestroyVgroup(pVgroup);
-      dnodeSendRpcMnodeWriteRsp(mnodeMsg, code);
+      dnodeSendRpcMWriteRsp(mnodeMsg, code);
     }
   } else {
     SSdbOper oper = {
@@ -979,7 +979,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
       .pObj = pVgroup
     };
     sdbDeleteRow(&oper);
-    dnodeSendRpcMnodeWriteRsp(mnodeMsg, mnodeMsg->code);
+    dnodeSendRpcMWriteRsp(mnodeMsg, mnodeMsg->code);
   }
 }
 
@@ -1041,7 +1041,7 @@ static void mnodeProcessDropVnodeRsp(SRpcMsg *rpcMsg) {
     code = TSDB_CODE_MND_SDB_ERROR;
   }
 
-  dnodeReprocessMnodeWriteMsg(mnodeMsg);
+  dnodeReprocessMWriteMsg(mnodeMsg);
 }
 
 static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) {

src/plugins/http/src/httpQueue.c

@@ -49,7 +49,7 @@ static taos_queue tsHttpQueue;
 
 void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t numOfRows, void (*fp)(void *param, void *result, int32_t numOfRows)) {
   if (tsHttpQueue != NULL) {
-    SHttpResult *pMsg = (SHttpResult *)taosAllocateQitem(sizeof(SHttpResult));
+    SHttpResult *pMsg = taosAllocateQitem(sizeof(SHttpResult));
     pMsg->param = param;
     pMsg->result = result;
     pMsg->numOfRows = numOfRows;

src/plugins/monitor/src/monitorMain.c

@@ -27,12 +27,12 @@
 #include "monitor.h"
 #include "taoserror.h"
 
-#define monitorFatal(...) { if (monitorDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }}
-#define monitorError(...) { if (monitorDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }}
-#define monitorWarn(...)  { if (monitorDebugFlag & DEBUG_WARN)  { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }}
-#define monitorInfo(...)  { if (monitorDebugFlag & DEBUG_INFO)  { taosPrintLog("MON ", 255, __VA_ARGS__); }}
-#define monitorDebug(...) { if (monitorDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
-#define monitorTrace(...) { if (monitorDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
+#define mnFatal(...) { if (monitorDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }}
+#define mnError(...) { if (monitorDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }}
+#define mnWarn(...)  { if (monitorDebugFlag & DEBUG_WARN)  { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }}
+#define mnInfo(...)  { if (monitorDebugFlag & DEBUG_INFO)  { taosPrintLog("MON ", 255, __VA_ARGS__); }}
+#define mnDebug(...) { if (monitorDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
+#define mnTrace(...) { if (monitorDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }}
 
 #define SQL_LENGTH     1030
 #define LOG_LEN_STR    100
@@ -91,12 +91,12 @@ int32_t monitorInitSystem() {
   pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
 
   if (pthread_create(&tsMonitor.thread, &thAttr, monitorThreadFunc, NULL)) {
-    monitorError("failed to create thread to for monitor module, reason:%s", strerror(errno));
+    mnError("failed to create thread to for monitor module, reason:%s", strerror(errno));
     return -1;
   }
 
   pthread_attr_destroy(&thAttr);
-  monitorDebug("monitor thread is launched");
+  mnDebug("monitor thread is launched");
 
   monitorStartSystemFp = monitorStartSystem;
   monitorStopSystemFp = monitorStopSystem;
@@ -107,12 +107,12 @@ int32_t monitorStartSystem() {
   taos_init();
   tsMonitor.start = 1;
   monitorExecuteSQLFp = monitorExecuteSQL;
-  monitorInfo("monitor module start");
+  mnInfo("monitor module start");
   return 0;
 }
 
 static void *monitorThreadFunc(void *param) {
-  monitorDebug("starting to initialize monitor module ...");
+  mnDebug("starting to initialize monitor module ...");
 
   while (1) {
     static int32_t accessTimes = 0;
@@ -121,7 +121,7 @@ static void *monitorThreadFunc(void *param) {
 
     if (tsMonitor.quiting) {
       tsMonitor.state = MON_STATE_NOT_INIT;
-      monitorInfo("monitor thread will quit, for taosd is quiting");
+      mnInfo("monitor thread will quit, for taosd is quiting");
       break;
     } else {
       taosGetDisk();
@@ -132,7 +132,7 @@ static void *monitorThreadFunc(void *param) {
     }
     
     if (dnodeGetDnodeId() <= 0) {
-      monitorDebug("dnode not initialized, waiting for 3000 ms to start monitor module");
+      mnDebug("dnode not initialized, waiting for 3000 ms to start monitor module");
       continue;
     }
 
@@ -140,10 +140,10 @@ static void *monitorThreadFunc(void *param) {
       tsMonitor.state = MON_STATE_NOT_INIT;
       tsMonitor.conn = taos_connect(NULL, "monitor", tsInternalPass, "", 0);
       if (tsMonitor.conn == NULL) {
-        monitorError("failed to connect to database, reason:%s", tstrerror(terrno));
+        mnError("failed to connect to database, reason:%s", tstrerror(terrno));
         continue;
       } else {
-        monitorDebug("connect to database success");
+        mnDebug("connect to database success");
       }
     }
 
@@ -155,10 +155,10 @@ static void *monitorThreadFunc(void *param) {
         taos_free_result(res);
 
         if (code != 0) {
-          monitorError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code));
+          mnError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code));
           break;
         } else {
-          monitorDebug("successfully to exec sql:%s", tsMonitor.sql);
+          mnDebug("successfully to exec sql:%s", tsMonitor.sql);
         }
       }
 
@@ -174,7 +174,7 @@ static void *monitorThreadFunc(void *param) {
     }
   }
 
-  monitorInfo("monitor thread is stopped");
+  mnInfo("monitor thread is stopped");
   return NULL;
 }
 
@@ -238,7 +238,7 @@ void monitorStopSystem() {
   tsMonitor.start = 0;
   tsMonitor.state = MON_STATE_NOT_INIT;
   monitorExecuteSQLFp = NULL;
-  monitorInfo("monitor module stopped");
+  mnInfo("monitor module stopped");
 }
 
 void monitorCleanUpSystem() {
@@ -249,7 +249,7 @@ void monitorCleanUpSystem() {
     taos_close(tsMonitor.conn);
     tsMonitor.conn = NULL;
   }
-  monitorInfo("monitor module is cleaned up");
+  mnInfo("monitor module is cleaned up");
 }
 
 // unit is MB
@@ -257,13 +257,13 @@ static int32_t monitorBuildMemorySql(char *sql) {
   float sysMemoryUsedMB = 0;
   bool  suc = taosGetSysMemory(&sysMemoryUsedMB);
   if (!suc) {
-    monitorDebug("failed to get sys memory info");
+    mnDebug("failed to get sys memory info");
   }
 
   float procMemoryUsedMB = 0;
   suc = taosGetProcMemory(&procMemoryUsedMB);
   if (!suc) {
-    monitorDebug("failed to get proc memory info");
+    mnDebug("failed to get proc memory info");
   }
 
   return sprintf(sql, ", %f, %f, %d", procMemoryUsedMB, sysMemoryUsedMB, tsTotalMemoryMB);
@@ -274,7 +274,7 @@ static int32_t monitorBuildCpuSql(char *sql) {
   float sysCpuUsage = 0, procCpuUsage = 0;
   bool  suc = taosGetCpuUsage(&sysCpuUsage, &procCpuUsage);
   if (!suc) {
-    monitorDebug("failed to get cpu usage");
+    mnDebug("failed to get cpu usage");
   }
 
   if (sysCpuUsage <= procCpuUsage) {
@@ -294,14 +294,14 @@ static int32_t monitorBuildBandSql(char *sql) {
   float bandSpeedKb = 0;
   bool  suc = taosGetBandSpeed(&bandSpeedKb);
   if (!suc) {
-    monitorDebug("failed to get bandwidth speed");
+    mnDebug("failed to get bandwidth speed");
   }
 
   return sprintf(sql, ", %f", bandSpeedKb);
 }
 
 static int32_t monitorBuildReqSql(char *sql) {
-  SDnodeStatisInfo info = dnodeGetStatisInfo();
+  SStatisInfo info = dnodeGetStatisInfo();
   return sprintf(sql, ", %d, %d, %d)", info.httpReqNum, info.queryReqNum, info.submitReqNum);
 }
 
@@ -309,7 +309,7 @@ static int32_t monitorBuildIoSql(char *sql) {
   float readKB = 0, writeKB = 0;
   bool  suc = taosGetProcIO(&readKB, &writeKB);
   if (!suc) {
-    monitorDebug("failed to get io info");
+    mnDebug("failed to get io info");
   }
 
   return sprintf(sql, ", %f, %f", readKB, writeKB);
@@ -332,19 +332,19 @@ static void monitorSaveSystemInfo() {
   taos_free_result(res);
 
   if (code != 0) {
-    monitorError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql);
+    mnError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql);
   } else {
-    monitorDebug("successfully to save system info, sql:%s", tsMonitor.sql);
+    mnDebug("successfully to save system info, sql:%s", tsMonitor.sql);
   }
 }
 
 static void montiorExecSqlCb(void *param, TAOS_RES *result, int32_t code) {
   int32_t c = taos_errno(result);
   if (c != TSDB_CODE_SUCCESS) {
-    monitorError("save %s failed, reason:%s", (char *)param, tstrerror(c));
+    mnError("save %s failed, reason:%s", (char *)param, tstrerror(c));
   } else {
     int32_t rows = taos_affected_rows(result);
-    monitorDebug("save %s succ, rows:%d", (char *)param, rows);
+    mnDebug("save %s succ, rows:%d", (char *)param, rows);
   }
 
   taos_free_result(result);
@@ -380,7 +380,7 @@ void monitorSaveAcctLog(SAcctMonitorObj *pMon) {
           pMon->totalConns, pMon->maxConns,
           pMon->accessState);
 
-  monitorDebug("save account info, sql:%s", sql);
+  mnDebug("save account info, sql:%s", sql);
   taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "account info");
 }
 
@@ -401,13 +401,13 @@ void monitorSaveLog(int32_t level, const char *const format, ...) {
   len += sprintf(sql + len, "', '%s')", tsLocalEp);
   sql[len++] = 0;
 
-  monitorDebug("save log, sql: %s", sql);
+  mnDebug("save log, sql: %s", sql);
   taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "log");
 }
 
 void monitorExecuteSQL(char *sql) {
   if (tsMonitor.state != MON_STATE_INITED) return;
 
-  monitorDebug("execute sql:%s", sql);
+  mnDebug("execute sql:%s", sql);
   taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "sql");
 }

src/query/inc/qExecutor.h

@@ -200,12 +200,6 @@ enum {
   QUERY_RESULT_READY     = 2,
 };
 
-typedef struct SMemRef {
-  int32_t ref; 
-  void *mem;
-  void *imem;
-} SMemRef;
-
 typedef struct SQInfo {
   void*            signature;
   int32_t          code;   // error code to returned to client

src/query/src/qExecutor.c

@@ -1711,6 +1711,19 @@ _clean:
   return TSDB_CODE_QRY_OUT_OF_MEMORY;
 }
 
+static void doFreeQueryHandle(SQInfo* pQInfo) {
+  SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
+
+  tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle);
+  tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
+
+  pRuntimeEnv->pQueryHandle = NULL;
+  pRuntimeEnv->pSecQueryHandle = NULL;
+
+  SMemRef* pMemRef = &pQInfo->memRef;
+  assert(pMemRef->ref == 0 && pMemRef->imem == NULL && pMemRef->mem == NULL);
+}
+
 static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
   if (pRuntimeEnv->pQuery == NULL) {
     return;
@@ -1740,8 +1753,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
   pRuntimeEnv->pFillInfo = taosDestroyFillInfo(pRuntimeEnv->pFillInfo);
 
   destroyResultBuf(pRuntimeEnv->pResultBuf);
-  tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle);
-  tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
+  doFreeQueryHandle(pQInfo);
 
   pRuntimeEnv->pTSBuf = tsBufDestroy(pRuntimeEnv->pTSBuf);
   taosTFree(pRuntimeEnv->keyBuf);
@@ -3536,7 +3548,7 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI
     tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
   }
 
-  pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
+  pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
   if (pRuntimeEnv->pSecQueryHandle == NULL) {
     longjmp(pRuntimeEnv->env, terrno);
   }
@@ -3620,7 +3632,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
     }
 
     restoreTimeWindow(&pQInfo->tableGroupInfo, &cond);
-    pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
+    pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
     if (pRuntimeEnv->pSecQueryHandle == NULL) {
       longjmp(pRuntimeEnv->env, terrno);
     }
@@ -4462,7 +4474,7 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery)
 
   terrno = TSDB_CODE_SUCCESS;
   if (isFirstLastRowQuery(pQuery)) {
-    pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
+    pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
 
     // update the query time window
     pQuery->window = cond.twindow;
@@ -4484,9 +4496,9 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery)
       }
     }
   } else if (isPointInterpoQuery(pQuery)) {
-    pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
+    pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
   } else {
-    pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
+    pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
   }
 
   return terrno;
@@ -4765,7 +4777,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) {
     pRuntimeEnv->pQueryHandle = NULL;
   }
 
-  pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo);
+  pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo, &pQInfo->memRef);
   taosArrayDestroy(tx);
   taosArrayDestroy(g1);
   if (pRuntimeEnv->pQueryHandle == NULL) {
@@ -4880,7 +4892,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
         pRuntimeEnv->pQueryHandle = NULL;
       }
 
-      pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo);
+      pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo, &pQInfo->memRef);
 
       taosArrayDestroy(tx);
       taosArrayDestroy(g1);
@@ -4946,7 +4958,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
       }
 
       // no need to update the lastkey for each table
-      pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo);
+      pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo, &pQInfo->memRef);
 
       taosArrayDestroy(g1);
       taosArrayDestroy(tx);
@@ -5155,7 +5167,7 @@ static void doSaveContext(SQInfo *pQInfo) {
   setupQueryRangeForReverseScan(pQInfo);
 
   pRuntimeEnv->prevGroupId = INT32_MIN;
-  pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
+  pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef);
   if (pRuntimeEnv->pSecQueryHandle == NULL) {
     longjmp(pRuntimeEnv->env, terrno);
   }
@@ -6841,7 +6853,7 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex
 
   int32_t code = TSDB_CODE_SUCCESS;
 
-#if 0
+#if _NON_BLOCKING_RETRIEVE
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
 
   pthread_mutex_lock(&pQInfo->lock);
@@ -6913,6 +6925,8 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
   pQInfo->dataReady  = QUERY_RESULT_NOT_READY;
 
   if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) {
+    // here current thread hold the refcount, so it is safe to free tsdbQueryHandle.
+    doFreeQueryHandle(pQInfo);
     *continueExec = false;
     (*pRsp)->completed = 1;  // notify no more result to client
   } else {

src/rpc/src/rpcMain.c

@@ -82,6 +82,7 @@ typedef struct {
   int8_t    oldInUse;   // server EP inUse passed by app
   int8_t    redirect;   // flag to indicate redirect
   int8_t    connType;   // connection type
+  int64_t   rid;        // refId returned by taosAddRef
   SRpcMsg  *pRsp;       // for synchronous API
   tsem_t   *pSem;       // for synchronous API
   SRpcEpSet *pSet;      // for synchronous API 
@@ -220,8 +221,7 @@ static void rpcFree(void *p) {
   free(p);
 }
 
-static void rpcInit(void) {
-
+void rpcInit(void) {
   tsProgressTimer = tsRpcTimer/2; 
   tsRpcMaxRetry = tsRpcMaxTime * 1000/tsProgressTimer;
   tsRpcHeadSize = RPC_MSG_OVERHEAD; 
@@ -230,6 +230,11 @@ static void rpcInit(void) {
   tsRpcRefId = taosOpenRef(200, rpcFree);
 }
  
+void rpcCleanup(void) {
+  taosCloseRef(tsRpcRefId);
+  tsRpcRefId = -1;
+}
+ 
 void *rpcOpen(const SRpcInit *pInit) {
   SRpcInfo *pRpc;
 
@@ -374,7 +379,7 @@ void *rpcReallocCont(void *ptr, int contLen) {
   return start + sizeof(SRpcReqContext) + sizeof(SRpcHead);
 }
 
-void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) {
+int64_t rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) {
   SRpcInfo       *pRpc = (SRpcInfo *)shandle;
   SRpcReqContext *pContext;
 
@@ -403,10 +408,11 @@ void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) {
   // set the handle to pContext, so app can cancel the request
   if (pMsg->handle) *((void **)pMsg->handle) = pContext;
 
-  taosAddRef(tsRpcRefId, pContext);
+  pContext->rid = taosAddRef(tsRpcRefId, pContext);
+
   rpcSendReqToServer(pRpc, pContext);
 
-  return;
+  return pContext->rid;
 }
 
 void rpcSendResponse(const SRpcMsg *pRsp) {
@@ -551,15 +557,14 @@ int rpcReportProgress(void *handle, char *pCont, int contLen) {
   return code;
 }
 
-void rpcCancelRequest(void *handle) {
-  SRpcReqContext *pContext = handle;
+void rpcCancelRequest(int64_t rid) {
 
-  int code = taosAcquireRef(tsRpcRefId, pContext);
-  if (code < 0) return;
+  SRpcReqContext *pContext = taosAcquireRef(tsRpcRefId, rid);
+  if (pContext == NULL) return;
 
   rpcCloseConn(pContext->pConn);
 
-  taosReleaseRef(tsRpcRefId, pContext);
+  taosReleaseRef(tsRpcRefId, rid);
 }
 
 static void rpcFreeMsg(void *msg) {
@@ -628,7 +633,7 @@ static void rpcReleaseConn(SRpcConn *pConn) {
     // if there is an outgoing message, free it
     if (pConn->outType && pConn->pReqMsg) {
       if (pConn->pContext) pConn->pContext->pConn = NULL; 
-      taosRemoveRef(tsRpcRefId, pConn->pContext);
+      taosRemoveRef(tsRpcRefId, pConn->pContext->rid);
     }
   }
 
@@ -1109,7 +1114,7 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) {
   }
 
   // free the request message
-  taosRemoveRef(tsRpcRefId, pContext); 
+  taosRemoveRef(tsRpcRefId, pContext->rid); 
 }
 
 static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqContext *pContext) {
@@ -1620,11 +1625,7 @@ static void rpcDecRef(SRpcInfo *pRpc)
     tDebug("%s rpc resources are released", pRpc->label);
     taosTFree(pRpc);
 
-    int count = atomic_sub_fetch_32(&tsRpcNum, 1);
-    if (count == 0) {
-      // taosCloseRef(tsRpcRefId);
-      // tsRpcInit = PTHREAD_ONCE_INIT;    // windows compliling error  
-    }
+    atomic_sub_fetch_32(&tsRpcNum, 1);
   }
 }
 

src/sync/inc/syncInt.h

@@ -141,6 +141,7 @@ typedef struct SSyncNode {
   int8_t       replica;
   int8_t       quorum;
   uint32_t     vgId;
+  int64_t      rid;
   void        *ahandle;
   int8_t       selfIndex;
   SSyncPeer   *peerInfo[TAOS_SYNC_MAX_REPLICA+1];  // extra one for arbitrator

src/sync/src/syncMain.c

@@ -142,14 +142,14 @@ void syncCleanUp() {
   sInfo("sync module is cleaned up");
 }
 
-void *syncStart(const SSyncInfo *pInfo) {
+int64_t syncStart(const SSyncInfo *pInfo) {
   const SSyncCfg *pCfg = &pInfo->syncCfg;
 
   SSyncNode *pNode = (SSyncNode *)calloc(sizeof(SSyncNode), 1);
   if (pNode == NULL) {
     sError("no memory to allocate syncNode");
     terrno = TAOS_SYSTEM_ERROR(errno);
-    return NULL;
+    return -1;
   }
 
   tstrncpy(pNode->path, pInfo->path, sizeof(pNode->path));
@@ -170,10 +170,10 @@ void *syncStart(const SSyncInfo *pInfo) {
   pNode->quorum = pCfg->quorum;
   if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica;
 
-  int ret = taosAddRef(tsSyncRefId, pNode);
-  if (ret < 0) {
+  pNode->rid = taosAddRef(tsSyncRefId, pNode);
+  if (pNode->rid < 0) {
     syncFreeNode(pNode);
-    return NULL;
+    return -1;
   }
 
   for (int i = 0; i < pCfg->replica; ++i) {
@@ -187,8 +187,8 @@ void *syncStart(const SSyncInfo *pInfo) {
   if (pNode->selfIndex < 0) {
     sInfo("vgId:%d, this node is not configured", pNode->vgId);
     terrno = TSDB_CODE_SYN_INVALID_CONFIG;
-    syncStop(pNode);
-    return NULL;
+    syncStop(pNode->rid);
+    return -1;
   }
 
   nodeVersion = pInfo->version;  // set the initial version
@@ -200,15 +200,15 @@ void *syncStart(const SSyncInfo *pInfo) {
   if (pNode->pSyncFwds == NULL) {
     sError("vgId:%d, no memory to allocate syncFwds", pNode->vgId);
     terrno = TAOS_SYSTEM_ERROR(errno);
-    syncStop(pNode);
-    return NULL;
+    syncStop(pNode->rid);
+    return -1;
   }
 
-  pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl);
+  pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, syncTmrCtrl);
   if (pNode->pFwdTimer == NULL) {
     sError("vgId:%d, failed to allocate timer", pNode->vgId);
-    syncStop(pNode);
-    return NULL;
+    syncStop(pNode->rid);
+    return -1;
   }
 
   syncAddArbitrator(pNode);
@@ -218,15 +218,14 @@ void *syncStart(const SSyncInfo *pInfo) {
     (*pNode->notifyRole)(pNode->ahandle, nodeRole);
   }
 
-  return pNode;
+  return pNode->rid;
 }
 
-void syncStop(void *param) {
-  SSyncNode *pNode = param;
+void syncStop(int64_t rid) {
   SSyncPeer *pPeer;
 
-  int ret = taosAcquireRef(tsSyncRefId, pNode);
-  if (ret < 0) return;
+  SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+  if (pNode == NULL) return;
 
   sInfo("vgId:%d, cleanup sync", pNode->vgId);
 
@@ -245,16 +244,15 @@ void syncStop(void *param) {
 
   pthread_mutex_unlock(&(pNode->mutex));
 
-  taosReleaseRef(tsSyncRefId, pNode);
-  taosRemoveRef(tsSyncRefId, pNode);
+  taosReleaseRef(tsSyncRefId, rid);
+  taosRemoveRef(tsSyncRefId, rid);
 }
 
-int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) {
-  SSyncNode *pNode = param;
+int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) {
   int        i, j;
 
-  int ret = taosAcquireRef(tsSyncRefId, pNode);
-  if (ret < 0) return TSDB_CODE_SYN_INVALID_CONFIG;
+  SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+  if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG;
 
   sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], pNewCfg->replica,
         pNode->replica);
@@ -318,29 +316,25 @@ int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) {
         syncRole[nodeRole]);
   syncBroadcastStatus(pNode);
 
-  taosReleaseRef(tsSyncRefId, pNode);
+  taosReleaseRef(tsSyncRefId, rid);
 
   return 0;
 }
 
-int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype) {
-  SSyncNode *pNode = param;
-
-  int ret = taosAcquireRef(tsSyncRefId, pNode);
-  if (ret < 0) return 0; 
+int32_t syncForwardToPeer(int64_t rid, void *data, void *mhandle, int qtype) {
+  SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+  if (pNode == NULL) return 0; 
 
   int32_t code = syncForwardToPeerImpl(pNode, data, mhandle, qtype);
 
-  taosReleaseRef(tsSyncRefId, pNode);
+  taosReleaseRef(tsSyncRefId, rid);
 
   return code;
 }
 
-void syncConfirmForward(void *param, uint64_t version, int32_t code) {
-  SSyncNode *pNode = param;
-
-  int ret = taosAcquireRef(tsSyncRefId, pNode);
-  if (ret < 0) return;
+void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {
+  SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+  if (pNode == NULL) return;
 
   SSyncPeer *pPeer = pNode->pMaster;
   if (pPeer && pNode->quorum > 1) {
@@ -365,15 +359,14 @@ void syncConfirmForward(void *param, uint64_t version, int32_t code) {
     }
   }
 
-  taosReleaseRef(tsSyncRefId, pNode);
+  taosReleaseRef(tsSyncRefId, rid);
 }
 
-void syncRecover(void *param) {
-  SSyncNode *pNode = param;
+void syncRecover(int64_t rid) {
   SSyncPeer *pPeer;
 
-  int ret = taosAcquireRef(tsSyncRefId, pNode);
-  if (ret < 0) return;
+  SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+  if (pNode == NULL) return;
 
   // to do: add a few lines to check if recover is OK
   // if take this node to unsync state, the whole system may not work
@@ -393,14 +386,12 @@ void syncRecover(void *param) {
 
   pthread_mutex_unlock(&(pNode->mutex));
 
-  taosReleaseRef(tsSyncRefId, pNode);
+  taosReleaseRef(tsSyncRefId, rid);
 }
 
-int syncGetNodesRole(void *param, SNodesRole *pNodesRole) {
-  SSyncNode *pNode = param;
-
-  int ret = taosAcquireRef(tsSyncRefId, pNode);
-  if (ret < 0) return -1;
+int syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) {
+  SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+  if (pNode == NULL) return -1;
 
   pNodesRole->selfIndex = pNode->selfIndex;
   for (int i = 0; i < pNode->replica; ++i) {
@@ -408,7 +399,7 @@ int syncGetNodesRole(void *param, SNodesRole *pNodesRole) {
     pNodesRole->role[i] = pNode->peerInfo[i]->role;
   }
 
-  taosReleaseRef(tsSyncRefId, pNode);
+  taosReleaseRef(tsSyncRefId, rid);
 
   return 0;
 }
@@ -455,7 +446,7 @@ void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_8(&pPeer->refCount, 1);
 
 int syncDecPeerRef(SSyncPeer *pPeer) {
   if (atomic_sub_fetch_8(&pPeer->refCount, 1) == 0) {
-    taosReleaseRef(tsSyncRefId, pPeer->pSyncNode);
+    taosReleaseRef(tsSyncRefId, pPeer->pSyncNode->rid);
 
     sDebug("%s, resource is freed", pPeer->id);
     taosTFree(pPeer->watchFd);
@@ -512,7 +503,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
     taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, syncTmrCtrl, &pPeer->timer);
   }
 
-  taosAcquireRef(tsSyncRefId, pNode);  
+  taosAcquireRef(tsSyncRefId, pNode->rid);  
   return pPeer;
 }
 
@@ -1105,7 +1096,7 @@ static void syncProcessBrokenLink(void *param) {
   SSyncPeer *pPeer = param;
   SSyncNode *pNode = pPeer->pSyncNode;
 
-  if (taosAcquireRef(tsSyncRefId, pNode) < 0) return;
+  if (taosAcquireRef(tsSyncRefId, pNode->rid) < 0) return;
   pthread_mutex_lock(&(pNode->mutex));
 
   sDebug("%s, TCP link is broken(%s)", pPeer->id, strerror(errno));
@@ -1116,7 +1107,7 @@ static void syncProcessBrokenLink(void *param) {
   }
 
   pthread_mutex_unlock(&(pNode->mutex));
-  taosReleaseRef(tsSyncRefId, pNode);
+  taosReleaseRef(tsSyncRefId, pNode->rid);
 }
 
 static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) {
@@ -1184,10 +1175,9 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code
 }
 
 static void syncMonitorFwdInfos(void *param, void *tmrId) {
-  SSyncNode *pNode = param;
-
-  int ret = taosAcquireRef(tsSyncRefId, pNode);
-  if ( ret < 0) return;
+  int64_t rid = (int64_t) param;
+  SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+  if (pNode == NULL) return;
 
   SSyncFwds *pSyncFwds = pNode->pSyncFwds;
 
@@ -1206,10 +1196,10 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) {
       pthread_mutex_unlock(&(pNode->mutex));
     }
 
-    pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl);
+    pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, syncTmrCtrl);
   }
 
-  taosReleaseRef(tsSyncRefId, pNode);
+  taosReleaseRef(tsSyncRefId, rid);
 }
 
 static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int qtype) {

src/sync/test/syncServer.c

@@ -30,7 +30,7 @@ int       dataFd = -1;
 void *    qhandle = NULL;
 int       walNum = 0;
 uint64_t  tversion = 0;
-void *    syncHandle;
+int64_t   syncHandle;
 int       role;
 int       nodeId;
 char      path[256];

src/tsdb/src/tsdbMemTable.c

@@ -198,6 +198,8 @@ void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemTable *pMem, SMemTable *pIMem)
   if (pIMem != NULL) {
     tsdbUnRefMemTable(pRepo, pIMem);
   }
+
+  tsdbDebug("vgId:%d utake memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pMem, pIMem);
 }
 
 void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) {

src/tsdb/src/tsdbRead.c

@@ -20,7 +20,6 @@
 #include "exception.h"
 
 #include "../../query/inc/qAst.h"  // todo move to common module
-#include "../../query/inc/qExecutor.h"  // todo move to common module
 #include "tlosertree.h"
 #include "tsdb.h"
 #include "tsdbMain.h"
@@ -120,8 +119,9 @@ typedef struct STsdbQueryHandle {
 
   SDataCols     *pDataCols;        // in order to hold current file data block
   int32_t        allocSize;        // allocated data block size
-  SMemTable     *mem;              // mem-table
-  SMemTable     *imem;             // imem-table, acquired from snapshot
+  SMemRef       *pMemRef;
+//  SMemTable     *mem;              // mem-table
+//  SMemTable     *imem;             // imem-table, acquired from snapshot
   SArray        *defaultLoadColumn;// default load column
   SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */
   SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */
@@ -184,26 +184,26 @@ static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS
   return pLocalIdList;
 }
 
-static void tsdbMayTakeMemSnapshot(TsdbQueryHandleT pHandle) { 
-  STsdbQueryHandle* pSecQueryHandle = (STsdbQueryHandle*) pHandle;
-  SQInfo *pQInfo = (SQInfo *)(pSecQueryHandle->qinfo);
+static void tsdbMayTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) {
+  assert(pQueryHandle != NULL && pQueryHandle->pMemRef != NULL);
 
-  if (pQInfo->memRef.ref++ == 0) {
-    tsdbTakeMemSnapshot(pSecQueryHandle->pTsdb, &pSecQueryHandle->mem, &pSecQueryHandle->imem);
-    pQInfo->memRef.mem = pSecQueryHandle->mem;
-    pQInfo->memRef.imem = pSecQueryHandle->imem;
-  } else {
-    pSecQueryHandle->mem = (SMemTable *)(pQInfo->memRef.mem);
-    pSecQueryHandle->imem = (SMemTable *)(pQInfo->memRef.imem);
+  SMemRef* pMemRef = pQueryHandle->pMemRef;
+  if (pQueryHandle->pMemRef->ref++ == 0) {
+    tsdbTakeMemSnapshot(pQueryHandle->pTsdb, (SMemTable**)&(pMemRef->mem), (SMemTable**)&(pMemRef->imem));
   }
 }
-static void tsdbMayUnTakeMemSnapshot(TsdbQueryHandleT pHandle) {
-  STsdbQueryHandle* pSecQueryHandle = (STsdbQueryHandle*) pHandle; 
-  SQInfo *pQInfo = (SQInfo *)(pSecQueryHandle->qinfo);
 
-  if (--pQInfo->memRef.ref == 0) {
-    tsdbUnTakeMemSnapShot(pSecQueryHandle->pTsdb, pSecQueryHandle->mem, pSecQueryHandle->imem);
+static void tsdbMayUnTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) {
+  assert(pQueryHandle != NULL && pQueryHandle->pMemRef != NULL);
+
+  SMemRef* pMemRef = pQueryHandle->pMemRef;
+  if (--pMemRef->ref == 0) {
+    tsdbUnTakeMemSnapShot(pQueryHandle->pTsdb, pMemRef->mem, pMemRef->imem);
+    pMemRef->mem = NULL;
+    pMemRef->imem = NULL;
   }
+
+  pQueryHandle->pMemRef = NULL;
 }
 static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STableGroupInfo* pGroupList, STsdbMeta* pMeta) {
   size_t sizeOfGroup = taosArrayGetSize(pGroupList->pGroupList);
@@ -270,7 +270,7 @@ static SArray* createCheckInfoFromCheckInfo(SArray* pTableCheckInfo, TSKEY skey)
   return pNew;
 }
 
-static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, void* qinfo) {
+static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, void* qinfo, SMemRef* pMemRef) {
   STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle));
   if (pQueryHandle == NULL) {
     goto out_of_memory;
@@ -288,13 +288,14 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond*
   pQueryHandle->outputCapacity = ((STsdbRepo*)tsdb)->config.maxRowsPerFileBlock;
   pQueryHandle->allocSize   = 0;
   pQueryHandle->locateStart = false;
+  pQueryHandle->pMemRef     = pMemRef;
 
   if (tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb) != 0) {
     goto out_of_memory;
   }
 
   tsdbMayTakeMemSnapshot(pQueryHandle);
-  assert(pCond != NULL && pCond->numOfCols > 0);
+  assert(pCond != NULL && pCond->numOfCols > 0 && pMemRef != NULL);
 
   if (ASCENDING_TRAVERSE(pCond->order)) {
     assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey);
@@ -348,8 +349,8 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond*
   return NULL;
 }
 
-TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo) {
-  STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qinfo);
+TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo, SMemRef* pRef) {
+  STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qinfo, pRef);
 
   STsdbMeta* pMeta = tsdbGetMeta(tsdb);
   assert(pMeta != NULL);
@@ -366,7 +367,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
   return (TsdbQueryHandleT) pQueryHandle;
 }
 
-TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) {
+TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pMemRef) {
   pCond->twindow = changeTableGroupByLastrow(groupList);
 
   // no qualified table
@@ -374,7 +375,7 @@ TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab
     return NULL;
   }
 
-  STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo);
+  STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo, pMemRef);
 
   assert(pCond->order == TSDB_ORDER_ASC && pCond->twindow.skey <= pCond->twindow.ekey);
   return pQueryHandle;
@@ -396,8 +397,8 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) {
   return res;
 }
 
-TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo) {
-  STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo);
+TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pRef) {
+  STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo, pRef);
   if (pQueryHandle != NULL) {
     pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL;
     changeQueryHandleForInterpQuery(pQueryHandle);
@@ -417,7 +418,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
   int32_t order = pHandle->order;
 
   // no data in buffer, abort
-  if (pHandle->mem == NULL && pHandle->imem == NULL) {
+  if (pHandle->pMemRef->mem == NULL && pHandle->pMemRef->imem == NULL) {
     return false;
   }
 
@@ -426,16 +427,19 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
   STableData* pMem = NULL;
   STableData* pIMem = NULL;
 
-  if (pHandle->mem && pCheckInfo->tableId.tid < pHandle->mem->maxTables) {
-    pMem = pHandle->mem->tData[pCheckInfo->tableId.tid];
+  SMemTable* pMemT = pHandle->pMemRef->mem;
+  SMemTable* pIMemT = pHandle->pMemRef->imem;
+
+  if (pMemT && pCheckInfo->tableId.tid < pMemT->maxTables) {
+    pMem = pMemT->tData[pCheckInfo->tableId.tid];
     if (pMem != NULL && pMem->uid == pCheckInfo->tableId.uid) { // check uid
       pCheckInfo->iter =
           tSkipListCreateIterFromVal(pMem->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order);
     }
   }
 
-  if (pHandle->imem && pCheckInfo->tableId.tid < pHandle->imem->maxTables) {
-    pIMem = pHandle->imem->tData[pCheckInfo->tableId.tid];
+  if (pIMemT && pCheckInfo->tableId.tid < pIMemT->maxTables) {
+    pIMem = pIMemT->tData[pCheckInfo->tableId.tid];
     if (pIMem != NULL && pIMem->uid == pCheckInfo->tableId.uid) { // check uid
       pCheckInfo->iiter =
           tSkipListCreateIterFromVal(pIMem->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order);
@@ -2029,7 +2033,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
         memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo));
       }
 
-      STsdbQueryHandle* pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qinfo);
+      STsdbQueryHandle* pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qinfo, pQueryHandle->pMemRef);
 
       taosTFree(cond.colList);
 

src/util/inc/tfile.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TFILE_H
+#define TDENGINE_TFILE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <unistd.h>
+
+// init taos file module
+int32_t tfinit();
+
+// clean up taos file module
+void tfcleanup();
+
+// the same syntax as UNIX standard open/close/read/write
+// but FD is int64_t and will never be reused
+int64_t tfopen(const char *pathname, int32_t flags);
+int64_t tfclose(int64_t tfd);
+int64_t tfwrite(int64_t tfd, void *buf, int64_t count);
+int64_t tfread(int64_t tfd, void *buf, int64_t count);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TREF_H

src/util/inc/tref.h

@@ -21,38 +21,48 @@
 extern "C" {
 #endif
 
-// open an instance, return refId which will be used by other APIs
-int  taosOpenRef(int max, void (*fp)(void *)); 
+// open a reference set, max is the mod used by hash, fp is the pointer to free resource function
+// return rsetId which will be used by other APIs. On error, -1 is returned, and terrno is set appropriately
+int taosOpenRef(int max, void (*fp)(void *)); 
 
-// close the Ref instance
-void taosCloseRef(int refId); 
+// close the reference set, refId is the return value by taosOpenRef
+// return 0 if success. On error, -1 is returned, and terrno is set appropriately
+int taosCloseRef(int refId); 
 
 // add ref, p is the pointer to resource or pointer ID
-int  taosAddRef(int refId, void *p);
-#define taosRemoveRef taosReleaseRef
+// return Reference ID(rid) allocated. On error, -1 is returned, and terrno is set appropriately
+int64_t taosAddRef(int refId, void *p);   
 
-// acquire ref, p is the pointer to resource or pointer ID
-int  taosAcquireRef(int refId, void *p);
+// remove ref, rid is the reference ID returned by taosAddRef
+// return 0 if success. On error, -1 is returned, and terrno is set appropriately
+int taosRemoveRef(int rsetId, int64_t rid);
 
-// release ref, p is the pointer to resource or pinter ID
-void taosReleaseRef(int refId, void *p);
+// acquire ref, rid is the reference ID returned by taosAddRef
+// return the resource p. On error, NULL is returned, and terrno is set appropriately
+void *taosAcquireRef(int rsetId, int64_t rid);
 
-// return the first if p is null, otherwise return the next after p
-void *taosIterateRef(int refId, void *p);
+// release ref, rid is the reference ID returned by taosAddRef
+// return 0 if success. On error, -1 is returned, and terrno is set appropriately
+int taosReleaseRef(int rsetId, int64_t rid);
+
+// return the first reference if rid is 0, otherwise return the next after current reference. 
+// if return value is NULL, it means list is over(if terrno is set, it means error happens)
+void *taosIterateRef(int rsetId, int64_t rid);
 
 // return the number of references in system
 int  taosListRef();  
 
 /* sample code to iterate the refs 
 
-void demoIterateRefs(int refId) {
+void demoIterateRefs(int rsetId) {
 
-  void *p = taosIterateRef(refId, NULL);
+  void *p = taosIterateRef(refId, 0);
   while (p) {
-
     // process P
+    
+    // get the rid from p
 
-    p = taosIterateRef(refId, p);
+    p = taosIterateRef(rsetId, rid);
   }
 }
 

src/util/src/tfile.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "os.h"
+#include "taoserror.h"
+#include "tulog.h"
+#include "tutil.h"
+#include "tref.h"
+
+static int32_t tsFileRsetId = -1;
+
+static void taosCloseFile(void *p) {
+  close((int32_t)(uintptr_t)p);
+}
+
+int32_t tfinit() {
+  tsFileRsetId = taosOpenRef(2000, taosCloseFile);
+  return tsFileRsetId;
+}
+
+void tfcleanup() {
+  if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId);
+  tsFileRsetId = -1;
+}
+
+int64_t tfopen(const char *pathname, int32_t flags) {
+  int32_t fd = open(pathname, flags);
+
+  if (fd < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  } 
+
+  void *p = (void *)(int64_t)fd;
+  int64_t rid = taosAddRef(tsFileRsetId, p);
+  if (rid < 0) close(fd);
+
+  return rid;
+}
+
+int64_t tfclose(int64_t tfd) {
+  return taosRemoveRef(tsFileRsetId, tfd);
+}
+
+int64_t tfwrite(int64_t tfd, void *buf, int64_t count) {
+  void *p = taosAcquireRef(tsFileRsetId, tfd);
+  if (p == NULL) return -1;
+
+  int32_t fd = (int32_t)(uintptr_t)p;
+
+  int64_t ret = taosWrite(fd, buf, count);
+  if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
+
+  taosReleaseRef(tsFileRsetId, tfd);
+  return ret;
+}
+
+int64_t tfread(int64_t tfd, void *buf, int64_t count) {
+  void *p = taosAcquireRef(tsFileRsetId, tfd);
+  if (p == NULL) return -1;
+
+  int32_t fd = (int32_t)(uintptr_t)p;
+
+  int64_t ret = taosRead(fd, buf, count);
+  if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno);
+
+  taosReleaseRef(tsFileRsetId, tfd);
+  return ret;
+}

src/util/src/tref.c

@@ -24,19 +24,22 @@
 #define TSDB_REF_STATE_DELETED 2
 
 typedef struct SRefNode {
-  struct SRefNode  *prev;
-  struct SRefNode  *next;
-  void             *p;
-  int32_t           count;
+  struct SRefNode  *prev;  // previous node 
+  struct SRefNode  *next;  // next node
+  void             *p;     // pointer to resource protected, 
+  int64_t           rid;   // reference ID
+  int32_t           count; // number of references
+  int               removed; // 1: removed 
 } SRefNode;
 	
 typedef struct {
-  SRefNode **nodeList;
-  int        state;  // 0: empty, 1: active;  2: deleted
-  int        refId;
-  int        max;
-  int32_t    count;  // total number of SRefNodes in this set
-  int64_t   *lockedBy;
+  SRefNode **nodeList; // array of SRefNode linked list
+  int        state;    // 0: empty, 1: active;  2: deleted
+  int        rsetId;   // refSet ID, global unique 
+  int64_t    rid;      // increase by one for each new reference
+  int        max;      // mod 
+  int32_t    count;    // total number of SRefNodes in this set
+  int64_t   *lockedBy; 
   void     (*fp)(void *);
 } SRefSet;
 
@@ -47,54 +50,58 @@ static int             tsRefSetNum = 0;
 static int             tsNextId = 0;
 
 static void taosInitRefModule(void);
-static int  taosHashRef(SRefSet *pSet, void *p); 
 static void taosLockList(int64_t *lockedBy);
 static void taosUnlockList(int64_t *lockedBy);
-static void taosIncRefCount(SRefSet *pSet);
-static void taosDecRefCount(SRefSet *pSet);
+static void taosIncRsetCount(SRefSet *pSet);
+static void taosDecRsetCount(SRefSet *pSet);
+static int  taosDecRefCount(int rsetId, int64_t rid, int remove);
 
 int taosOpenRef(int max, void (*fp)(void *))
 {
   SRefNode **nodeList;
   SRefSet   *pSet;
   int64_t   *lockedBy;
-  int        i, refId;
+  int        i, rsetId;
   
   pthread_once(&tsRefModuleInit, taosInitRefModule);
    
   nodeList = calloc(sizeof(SRefNode *), (size_t)max);
-  if (nodeList == NULL) {
-    return TSDB_CODE_REF_NO_MEMORY;
+  if (nodeList == NULL)  {
+    terrno = TSDB_CODE_REF_NO_MEMORY;
+    return -1;
   }
 
   lockedBy = calloc(sizeof(int64_t), (size_t)max);
   if (lockedBy == NULL) {
     free(nodeList);
-    return TSDB_CODE_REF_NO_MEMORY;
+    terrno = TSDB_CODE_REF_NO_MEMORY;
+    return -1;
   }
 
   pthread_mutex_lock(&tsRefMutex);
  
   for (i = 0; i < TSDB_REF_OBJECTS; ++i) {
     tsNextId = (tsNextId + 1) % TSDB_REF_OBJECTS;
+    if (tsNextId == 0) tsNextId = 1;   // dont use 0 as rsetId
     if (tsRefSetList[tsNextId].state == TSDB_REF_STATE_EMPTY) break;
   } 
 
   if (i < TSDB_REF_OBJECTS) {
-    refId = tsNextId;
-    pSet = tsRefSetList + refId;
-    taosIncRefCount(pSet);
+    rsetId = tsNextId;
+    pSet = tsRefSetList + rsetId;
     pSet->max = max;
     pSet->nodeList = nodeList;
     pSet->lockedBy = lockedBy;
     pSet->fp = fp;
+    pSet->rid = 1;
+    pSet->rsetId = rsetId;
     pSet->state = TSDB_REF_STATE_ACTIVE;
-    pSet->refId = refId;
+    taosIncRsetCount(pSet);
 
     tsRefSetNum++;
-    uTrace("refId:%d is opened, max:%d, fp:%p refSetNum:%d", refId, max, fp, tsRefSetNum);
+    uTrace("rsetId:%d is opened, max:%d, fp:%p refSetNum:%d", rsetId, max, fp, tsRefSetNum);
   } else {
-    refId = TSDB_CODE_REF_FULL;
+    rsetId = TSDB_CODE_REF_FULL;
     free (nodeList);
     free (lockedBy);
     uTrace("run out of Ref ID, maximum:%d refSetNum:%d", TSDB_REF_OBJECTS, tsRefSetNum);
@@ -102,121 +109,128 @@ int taosOpenRef(int max, void (*fp)(void *))
 
   pthread_mutex_unlock(&tsRefMutex);
 
-  return refId;
+  return rsetId;
 }
 
-void taosCloseRef(int refId)
+int taosCloseRef(int rsetId)
 {
   SRefSet  *pSet;
   int       deleted = 0;
 
-  if (refId < 0 || refId >= TSDB_REF_OBJECTS) {
-    uTrace("refId:%d is invalid, out of range", refId);
-    return;
+  if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
+    uTrace("rsetId:%d is invalid, out of range", rsetId);
+    terrno = TSDB_CODE_REF_INVALID_ID;
+    return -1;
   }
 
-  pSet = tsRefSetList + refId;
+  pSet = tsRefSetList + rsetId;
 
   pthread_mutex_lock(&tsRefMutex);
 
   if (pSet->state == TSDB_REF_STATE_ACTIVE) { 
     pSet->state = TSDB_REF_STATE_DELETED;
     deleted = 1;
-    uTrace("refId:%d is closed, count:%d", refId, pSet->count);
+    uTrace("rsetId:%d is closed, count:%d", rsetId, pSet->count);
   } else {
-    uTrace("refId:%d is already closed, count:%d", refId, pSet->count);
+    uTrace("rsetId:%d is already closed, count:%d", rsetId, pSet->count);
   }
 
   pthread_mutex_unlock(&tsRefMutex);
 
-  if (deleted) taosDecRefCount(pSet);
+  if (deleted) taosDecRsetCount(pSet);
+
+  return 0;
 }
 
-int taosAddRef(int refId, void *p) 
+int64_t taosAddRef(int rsetId, void *p) 
 {
   int       hash;
   SRefNode *pNode;
   SRefSet  *pSet;
+  int64_t   rid = 0;
 
-  if (refId < 0 || refId >= TSDB_REF_OBJECTS) {
-    uTrace("refId:%d p:%p failed to add, refId not valid", refId, p);
-    return TSDB_CODE_REF_INVALID_ID;
+  if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
+    uTrace("rsetId:%d p:%p failed to add, rsetId not valid", rsetId, p);
+    terrno = TSDB_CODE_REF_INVALID_ID;
+    return -1;
   }
 
-  pSet = tsRefSetList + refId;
-  taosIncRefCount(pSet);
+  pSet = tsRefSetList + rsetId;
+  taosIncRsetCount(pSet);
   if (pSet->state != TSDB_REF_STATE_ACTIVE) {
-    taosDecRefCount(pSet);
-    uTrace("refId:%d p:%p failed to add, not active", refId, p);
-    return TSDB_CODE_REF_ID_REMOVED;
+    taosDecRsetCount(pSet);
+    uTrace("rsetId:%d p:%p failed to add, not active", rsetId, p);
+    terrno = TSDB_CODE_REF_ID_REMOVED;
+    return -1;
   }
   
-  int code = 0;
-  hash = taosHashRef(pSet, p);
+  pNode = calloc(sizeof(SRefNode), 1);
+  if (pNode == NULL) {
+    terrno = TSDB_CODE_REF_NO_MEMORY;
+    return -1;
+  }
 
+  rid = atomic_add_fetch_64(&pSet->rid, 1);
+  hash = rid % pSet->max;
   taosLockList(pSet->lockedBy+hash);
   
-  pNode = pSet->nodeList[hash];
-  while (pNode) {
-    if (pNode->p == p)
-      break;
+  pNode->p = p;
+  pNode->rid = rid;
+  pNode->count = 1;
 
-    pNode = pNode->next;  
-  }
-
-  if (pNode) {
-    code = TSDB_CODE_REF_ALREADY_EXIST;
-    uTrace("refId:%d p:%p is already there, faild to add", refId, p);
-  } else {
-    pNode = calloc(sizeof(SRefNode), 1);
-    if (pNode) {
-      pNode->p = p;
-      pNode->count = 1;
-      pNode->prev = 0;
-      pNode->next = pSet->nodeList[hash];
-      if (pSet->nodeList[hash]) pSet->nodeList[hash]->prev = pNode;
-      pSet->nodeList[hash] = pNode;
-      uTrace("refId:%d p:%p is added, count:%d  malloc mem: %p", refId, p, pSet->count, pNode);
-    } else {
-      code = TSDB_CODE_REF_NO_MEMORY;
-      uTrace("refId:%d p:%p is not added, since no memory", refId, p);
-    }
-  }
+  pNode->prev = NULL;
+  pNode->next = pSet->nodeList[hash];
+  if (pSet->nodeList[hash]) pSet->nodeList[hash]->prev = pNode;
+  pSet->nodeList[hash] = pNode;
 
-  if (code < 0) taosDecRefCount(pSet);
+  uTrace("rsetId:%d p:%p rid:%" PRId64 " is added, count:%d", rsetId, p, rid, pSet->count);
 
   taosUnlockList(pSet->lockedBy+hash);
 
-  return code; 
+  return rid; 
+}
+
+int taosRemoveRef(int rsetId, int64_t rid) 
+{
+  return taosDecRefCount(rsetId, rid, 1);
 }
 
-int taosAcquireRef(int refId, void *p) 
+// if rid is 0, return the first p in hash list, otherwise, return the next after current rid
+void *taosAcquireRef(int rsetId, int64_t rid) 
 {
-  int       hash, code = 0;
+  int       hash;
   SRefNode *pNode;
   SRefSet  *pSet;
+  void     *p = NULL;
 
-  if (refId < 0 || refId >= TSDB_REF_OBJECTS) {
-    uTrace("refId:%d p:%p failed to acquire, refId not valid", refId, p);
-    return TSDB_CODE_REF_INVALID_ID;
+  if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
+    uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rsetId not valid", rsetId, rid);
+    terrno = TSDB_CODE_REF_INVALID_ID;
+    return NULL;
   }
 
-  pSet = tsRefSetList + refId;
-  taosIncRefCount(pSet);
+  if (rid <= 0) {
+    uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rid not valid", rsetId, rid);
+    terrno = TSDB_CODE_REF_NOT_EXIST;
+    return NULL;
+  }
+
+  pSet = tsRefSetList + rsetId;
+  taosIncRsetCount(pSet);
   if (pSet->state != TSDB_REF_STATE_ACTIVE) {
-    uTrace("refId:%d p:%p failed to acquire, not active", refId, p);
-    taosDecRefCount(pSet);
-    return TSDB_CODE_REF_ID_REMOVED;
+    uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, not active", rsetId, rid);
+    taosDecRsetCount(pSet);
+    terrno = TSDB_CODE_REF_ID_REMOVED;
+    return NULL;
   }
   
-  hash = taosHashRef(pSet, p);
-
+  hash = rid % pSet->max;
   taosLockList(pSet->lockedBy+hash);
 
   pNode = pSet->nodeList[hash];
 
   while (pNode) {
-    if (pNode->p == p) {
+    if (pNode->rid == rid) {
       break;
     }
       
@@ -224,117 +238,76 @@ int taosAcquireRef(int refId, void *p)
   }
 
   if (pNode) {
-    pNode->count++;
-    uTrace("refId:%d p:%p is acquired", refId, p);
+    if (pNode->removed == 0) {
+      pNode->count++;
+      p = pNode->p;
+      uTrace("rsetId:%d p:%p rid:%" PRId64 " is acquired", rsetId, pNode->p, rid);
+    } else {
+      terrno = TSDB_CODE_REF_NOT_EXIST;
+      uTrace("rsetId:%d p:%p rid:%" PRId64 " is already removed, failed to acquire", rsetId, pNode->p, rid);
+    }
   } else {
-    code = TSDB_CODE_REF_NOT_EXIST;
-    uTrace("refId:%d p:%p is not there, failed to acquire", refId, p);
+    terrno = TSDB_CODE_REF_NOT_EXIST;
+    uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to acquire", rsetId, rid);
   }
 
   taosUnlockList(pSet->lockedBy+hash);
 
-  taosDecRefCount(pSet);
+  taosDecRsetCount(pSet);
 
-  return code;
+  return p;
 }
 
-void taosReleaseRef(int refId, void *p) 
+int taosReleaseRef(int rsetId, int64_t rid) 
 {
-  int       hash;
-  SRefNode *pNode;
-  SRefSet  *pSet;
-  int       released = 0;
-
-  if (refId < 0 || refId >= TSDB_REF_OBJECTS) {
-    uTrace("refId:%d p:%p failed to release, refId not valid", refId, p);
-    return;
-  }
-
-  pSet = tsRefSetList + refId;
-  if (pSet->state == TSDB_REF_STATE_EMPTY) {
-    uTrace("refId:%d p:%p failed to release, cleaned", refId, p);
-    return;
-  }
-  
-  hash = taosHashRef(pSet, p);
-
-  taosLockList(pSet->lockedBy+hash);
-  
-  pNode = pSet->nodeList[hash];
-  while (pNode) {
-    if (pNode->p == p)
-      break;
-
-    pNode = pNode->next;  
-  }
-
-  if (pNode) {
-    pNode->count--;
-
-    if (pNode->count == 0) {
-      if ( pNode->prev ) {
-        pNode->prev->next = pNode->next;
-      } else {
-        pSet->nodeList[hash] = pNode->next;  
-      }
-		
-      if ( pNode->next ) {
-        pNode->next->prev = pNode->prev; 
-      } 
-		
-      (*pSet->fp)(pNode->p); 
-
-      free(pNode);
-      released = 1;
-      uTrace("refId:%d p:%p is removed, count:%d, free mem: %p", refId, p, pSet->count, pNode);
-    } else {
-      uTrace("refId:%d p:%p is released", refId, p);
-    }
-  } else {
-    uTrace("refId:%d p:%p is not there, failed to release", refId, p);
-  }
-
-  taosUnlockList(pSet->lockedBy+hash);
-
-  if (released) taosDecRefCount(pSet);
+  return taosDecRefCount(rsetId, rid, 0);
 }
 
-// if p is NULL, return the first p in hash list, otherwise, return the next after p
-void *taosIterateRef(int refId, void *p) {
+// if rid is 0, return the first p in hash list, otherwise, return the next after current rid
+void *taosIterateRef(int rsetId, int64_t rid) {
   SRefNode *pNode = NULL;
   SRefSet  *pSet;
 
-  if (refId < 0 || refId >= TSDB_REF_OBJECTS) {
-    uTrace("refId:%d p:%p failed to iterate, refId not valid", refId, p);
+  if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
+    uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rsetId not valid", rsetId, rid);
+    terrno = TSDB_CODE_REF_INVALID_ID;
+    return NULL;
+  }
+
+  if (rid <= 0) {
+    uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rid not valid", rsetId, rid);
+    terrno = TSDB_CODE_REF_NOT_EXIST;
     return NULL;
   }
 
-  pSet = tsRefSetList + refId;
-  taosIncRefCount(pSet);
+  pSet = tsRefSetList + rsetId;
+  taosIncRsetCount(pSet);
   if (pSet->state != TSDB_REF_STATE_ACTIVE) {
-    uTrace("refId:%d p:%p failed to iterate, not active", refId, p);
-    taosDecRefCount(pSet);
+    uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rset not active", rsetId, rid);
+    terrno = TSDB_CODE_REF_ID_REMOVED;
+    taosDecRsetCount(pSet);
     return NULL;
   }
 
   int hash = 0;
-  if (p) {
-    hash = taosHashRef(pSet, p);
+  if (rid > 0) {
+    hash = rid % pSet->max;
     taosLockList(pSet->lockedBy+hash);
 
     pNode = pSet->nodeList[hash];
     while (pNode) {
-      if (pNode->p == p) break;
+      if (pNode->rid == rid) break;
       pNode = pNode->next;   
     }
 
     if (pNode == NULL) {
-      uError("refId:%d p:%p not there, quit", refId, p);
+      uError("rsetId:%d rid:%" PRId64 " not there, quit", rsetId, rid);
+      terrno = TSDB_CODE_REF_NOT_EXIST;
       taosUnlockList(pSet->lockedBy+hash);
       return NULL;
     }
 
-    // p is there
+    // rid is there
     pNode = pNode->next;
     if (pNode == NULL) { 
       taosUnlockList(pSet->lockedBy+hash);
@@ -356,14 +329,14 @@ void *taosIterateRef(int refId, void *p) {
     pNode->count++;  // acquire it
     newP = pNode->p;  
     taosUnlockList(pSet->lockedBy+hash);
-    uTrace("refId:%d p:%p is returned", refId, p);
+    uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid);
   } else {
-    uTrace("refId:%d p:%p the list is over", refId, p);
+    uTrace("rsetId:%d the list is over", rsetId);
   }
 
-  if (p) taosReleaseRef(refId, p);  // release the current one
+  if (rid > 0) taosReleaseRef(rsetId, rid);  // release the current one
 
-  taosDecRefCount(pSet);
+  taosDecRsetCount(pSet);
 
   return newP;
 }
@@ -381,13 +354,13 @@ int taosListRef() {
     if (pSet->state == TSDB_REF_STATE_EMPTY) 
       continue;
 
-    uInfo("refId:%d state:%d count::%d", i, pSet->state, pSet->count);
+    uInfo("rsetId:%d state:%d count::%d", i, pSet->state, pSet->count);
 
     for (int j=0; j < pSet->max; ++j) {
       pNode = pSet->nodeList[j];
      
       while (pNode) {
-        uInfo("refId:%d p:%p count:%d", i, pNode->p, pNode->count);
+        uInfo("rsetId:%d p:%p rid:%" PRId64 "count:%d", i, pNode->p, pNode->rid, pNode->count);
         pNode = pNode->next;
         num++;
       }
@@ -399,21 +372,78 @@ int taosListRef() {
   return num;
 }
 
-static int taosHashRef(SRefSet *pSet, void *p) 
-{
-  int     hash = 0;
-  int64_t v = (int64_t)p;
+static int taosDecRefCount(int rsetId, int64_t rid, int remove) {
+  int       hash;
+  SRefSet  *pSet;
+  SRefNode *pNode;
+  int       released = 0;
+  int       code = 0;
+
+  if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) {
+    uTrace("rsetId:%d rid:%" PRId64 " failed to remove, rsetId not valid", rsetId, rid);
+    terrno = TSDB_CODE_REF_INVALID_ID;
+    return -1;
+  }
+
+  if (rid <= 0) {
+    uTrace("rsetId:%d rid:%" PRId64 " failed to remove, rid not valid", rsetId, rid);
+    terrno = TSDB_CODE_REF_NOT_EXIST;
+    return -1;
+  }
+
+  pSet = tsRefSetList + rsetId;
+  if (pSet->state == TSDB_REF_STATE_EMPTY) {
+    uTrace("rsetId:%d rid:%" PRId64 " failed to remove, cleaned", rsetId, rid);
+    terrno = TSDB_CODE_REF_ID_REMOVED;
+    return -1;
+  }
   
-  for (int i = 0; i < sizeof(v); ++i) {
-    hash += (int)(v & 0xFFFF);
-    v = v >> 16;
-    i = i + 2;
+  hash = rid % pSet->max;
+  taosLockList(pSet->lockedBy+hash);
+  
+  pNode = pSet->nodeList[hash];
+  while (pNode) {
+    if (pNode->rid == rid)
+      break;
+
+    pNode = pNode->next;  
   }
 
-  hash = hash % pSet->max;
+  if (pNode) {
+    pNode->count--;
+    if (remove) pNode->removed = 1;
 
-  return hash;
-} 
+    if (pNode->count <= 0) {
+      if (pNode->prev) {
+        pNode->prev->next = pNode->next;
+      } else {
+        pSet->nodeList[hash] = pNode->next;  
+      }
+		
+      if (pNode->next) {
+        pNode->next->prev = pNode->prev; 
+      } 
+		
+      (*pSet->fp)(pNode->p); 
+
+      uTrace("rsetId:%d p:%p rid:%" PRId64 "is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode);
+      free(pNode);
+      released = 1;
+    } else {
+      uTrace("rsetId:%d p:%p rid:%" PRId64 "is released, count:%d", rsetId, pNode->p, rid, pNode->count);
+    }
+  } else {
+    uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid);
+    terrno = TSDB_CODE_REF_NOT_EXIST;
+    code = -1;
+  }
+
+  taosUnlockList(pSet->lockedBy+hash);
+
+  if (released) taosDecRsetCount(pSet);
+
+  return code;
+}
 
 static void taosLockList(int64_t *lockedBy) {
   int64_t tid = taosGetPthreadId();
@@ -436,14 +466,14 @@ static void taosInitRefModule(void) {
   pthread_mutex_init(&tsRefMutex, NULL);
 }
 
-static void taosIncRefCount(SRefSet *pSet) {
+static void taosIncRsetCount(SRefSet *pSet) {
   atomic_add_fetch_32(&pSet->count, 1);
-  uTrace("refId:%d inc count:%d", pSet->refId, pSet->count);
+  // uTrace("rsetId:%d inc count:%d", pSet->rsetId, count);
 }
 
-static void taosDecRefCount(SRefSet *pSet) {
+static void taosDecRsetCount(SRefSet *pSet) {
   int32_t count = atomic_sub_fetch_32(&pSet->count, 1);
-  uTrace("refId:%d dec count:%d", pSet->refId, pSet->count);
+  // uTrace("rsetId:%d dec count:%d", pSet->rsetId, count);
 
   if (count > 0) return;
 
@@ -458,7 +488,7 @@ static void taosDecRefCount(SRefSet *pSet) {
     taosTFree(pSet->lockedBy);
 
     tsRefSetNum--;
-    uTrace("refId:%d is cleaned, refSetNum:%d count:%d", pSet->refId, tsRefSetNum, pSet->count);
+    uTrace("rsetId:%d is cleaned, refSetNum:%d count:%d", pSet->rsetId, tsRefSetNum, pSet->count);
   }
 
   pthread_mutex_unlock(&tsRefMutex);

src/util/tests/trefTest.c

@@ -11,106 +11,119 @@
 #include "tulog.h"
 
 typedef struct {
-  int  refNum;
-  int  steps;
-  int  refId;
-  void **p;
+  int     refNum;
+  int     steps;
+  int     rsetId;
+  int64_t rid;
+  void  **p;
 } SRefSpace;
 
-void iterateRefs(int refId) {
+void iterateRefs(int rsetId) {
   int  count = 0;
 
-  void *p = taosIterateRef(refId, NULL);
+  void *p = taosIterateRef(rsetId, NULL);
   while (p) {
     // process P
     count++;
-    p = taosIterateRef(refId, p);
+    p = taosIterateRef(rsetId, p);
   }    
 
   printf(" %d ", count); 
 }
 
-void *takeRefActions(void *param) {
+void *addRef(void *param) {
   SRefSpace *pSpace = (SRefSpace *)param;
-  int code, id;
+  int id;
+  int64_t rid;
 
   for (int i=0; i < pSpace->steps; ++i) {
-    printf("s");
+    printf("a");
     id = random() % pSpace->refNum; 
-    code = taosAddRef(pSpace->refId, pSpace->p[id]);
-    usleep(1);
-    
-    id = random() % pSpace->refNum; 
-    code = taosAcquireRef(pSpace->refId, pSpace->p[id]);
-    if (code >= 0) {
-      usleep(id % 5 + 1);
-      taosReleaseRef(pSpace->refId, pSpace->p[id]);
+    if (pSpace->rid[id] <= 0) {
+      pSpace->p[id] = malloc(128);
+      pSpace->rid[id] = taosAddRef(pSpace->rsetId, pSpace->p[id]);
     }
+    usleep(100);
+  }  
 
+  return NULL;
+}
+       
+void *removeRef(void *param) {
+  SRefSpace *pSpace = (SRefSpace *)param;
+  int id;
+  int64_t rid;
+
+  for (int i=0; i < pSpace->steps; ++i) {
+    printf("d");
     id = random() % pSpace->refNum; 
-    taosRemoveRef(pSpace->refId, pSpace->p[id]);
-    usleep(id %5 + 1);
+    if (pSpace->rid[id] > 0) {
+      code = taosRemoveRef(pSpace->rsetId, pSpace->rid[id]);
+      if (code == 0) pSpace->rid[id] = 0;
+    }
 
+    usleep(100);
+  }  
+
+  return NULL;
+}
+       
+void *acquireRelease(void *param) {
+  SRefSpace *pSpace = (SRefSpace *)param;
+  int id;
+  int64_t rid;
+
+  for (int i=0; i < pSpace->steps; ++i) {
+    printf("a");
+    
     id = random() % pSpace->refNum; 
-    code = taosAcquireRef(pSpace->refId, pSpace->p[id]);
+    code = taosAcquireRef(pSpace->rsetId, pSpace->p[id]);
     if (code >= 0) {
       usleep(id % 5 + 1);
-      taosReleaseRef(pSpace->refId, pSpace->p[id]);
+      taosReleaseRef(pSpace->rsetId, pSpace->p[id]);
     }
-
-    id = random() % pSpace->refNum; 
-    iterateRefs(id);
   }  
 
-  for (int i=0; i < pSpace->refNum; ++i) {
-    taosRemoveRef(pSpace->refId, pSpace->p[i]);
-  }
-  
-  //uInfo("refId:%d thread exits", pSpace->refId);
-
   return NULL;
 }
        
 void myfree(void *p) {
-  return;
+  free(p);
 }
 
 void *openRefSpace(void *param) {
   SRefSpace *pSpace = (SRefSpace *)param;
 
   printf("c");
-  pSpace->refId = taosOpenRef(50, myfree);
+  pSpace->rsetId = taosOpenRef(50, myfree);
 
-  if (pSpace->refId < 0) {
-    printf("failed to open ref, reson:%s\n", tstrerror(pSpace->refId));
+  if (pSpace->rsetId < 0) {
+    printf("failed to open ref, reson:%s\n", tstrerror(pSpace->rsetId));
     return NULL;
   } 
 
   pSpace->p = (void **) calloc(sizeof(void *), pSpace->refNum);
-  for (int i=0; i<pSpace->refNum; ++i) {
-    pSpace->p[i] = (void *) malloc(128);
-  }
 
   pthread_attr_t thattr;
   pthread_attr_init(&thattr);
   pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
 
   pthread_t thread1, thread2, thread3;
-  pthread_create(&(thread1), &thattr, takeRefActions, (void *)(pSpace));
-  pthread_create(&(thread2), &thattr, takeRefActions, (void *)(pSpace));
-  pthread_create(&(thread3), &thattr, takeRefActions, (void *)(pSpace));
+  pthread_create(&(thread1), &thattr, addRef, (void *)(pSpace));
+  pthread_create(&(thread2), &thattr, removeRef, (void *)(pSpace));
+  pthread_create(&(thread3), &thattr, acquireRelease, (void *)(pSpace));
 
   pthread_join(thread1, NULL);
   pthread_join(thread2, NULL);
   pthread_join(thread3, NULL);
 
-  taosCloseRef(pSpace->refId);
-
   for (int i=0; i<pSpace->refNum; ++i) {
-    free(pSpace->p[i]);
+    taosRemoveRef(pSpace->rsetId, pSpace->rid[i]);
   }
 
-  uInfo("refId:%d main thread exit", pSpace->refId);
+  taosCloseRef(pSpace->rsetId);
+
+  uInfo("rsetId:%d main thread exit", pSpace->rsetId);
   free(pSpace->p);
   pSpace->p = NULL;
 
@@ -140,7 +153,7 @@ int main(int argc, char *argv[]) {
       printf("\nusage: %s [options] \n", argv[0]);
       printf("  [-n]: number of references, default: %d\n", refNum);
       printf("  [-s]: steps to run for each reference, default: %d\n", steps);
-      printf("  [-t]: number of refIds running in parallel, default: %d\n", threads);
+      printf("  [-t]: number of rsetIds running in parallel, default: %d\n", threads);
       printf("  [-l]: number of loops, default: %d\n", loops);
       printf("  [-d]: debugFlag, default: %d\n", uDebugFlag);
       exit(0);

src/vnode/inc/vnodeInt.h

@@ -41,13 +41,13 @@ typedef struct {
   int8_t       status; 
   int8_t       role;   
   int8_t       accessState;
-  int64_t      version;   // current version 
-  int64_t      fversion;  // version on saved data file
+  uint64_t     version;   // current version 
+  uint64_t     fversion;  // version on saved data file
   void        *wqueue;
   void        *rqueue;
   void        *wal;
   void        *tsdb;
-  void        *sync;
+  int64_t      sync;
   void        *events;
   void        *cq;  // continuous query
   int32_t      cfgVersion;

src/vnode/src/vnodeCfg.c

@@ -22,7 +22,6 @@
 #include "tsdb.h"
 #include "dnode.h"
 #include "vnodeInt.h"
-#include "vnodeVersion.h"
 #include "vnodeCfg.h"
 
 static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) {
@@ -40,8 +39,7 @@ static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) {
   pVnode->tsdbCfg.compression = vnodeMsg->cfg.compression;
   pVnode->walCfg.walLevel = vnodeMsg->cfg.walLevel;
   pVnode->walCfg.fsyncPeriod = vnodeMsg->cfg.fsyncPeriod;
-  pVnode->walCfg.wals = vnodeMsg->cfg.wals;
-  pVnode->walCfg.keep = 0;
+  pVnode->walCfg.keep = TAOS_WAL_NOT_KEEP;
   pVnode->syncCfg.replica = vnodeMsg->cfg.replications;
   pVnode->syncCfg.quorum = vnodeMsg->cfg.quorum;
 

src/vnode/src/vnodeMain.c

@@ -15,17 +15,11 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-
-#include "tcache.h"
-#include "cJSON.h"
-#include "dnode.h"
-#include "hash.h"
 #include "taoserror.h"
 #include "taosmsg.h"
 #include "tglobal.h"
 #include "trpc.h"
 #include "tsdb.h"
-#include "ttimer.h"
 #include "tutil.h"
 #include "vnode.h"
 #include "vnodeInt.h"
@@ -34,7 +28,7 @@
 #include "vnodeCfg.h"
 #include "vnodeVersion.h"
 
-static SHashObj*tsDnodeVnodesHash;
+static SHashObj*tsVnodesHash;
 static void     vnodeCleanUp(SVnodeObj *pVnode);
 static int      vnodeProcessTsdbStatus(void *arg, int status);
 static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion);
@@ -44,12 +38,12 @@ static void     vnodeCtrlFlow(void *handle, int32_t mseconds);
 static int      vnodeNotifyFileSynced(void *ahandle, uint64_t fversion);
 
 #ifndef _SYNC
-tsync_h syncStart(const SSyncInfo *info) { return NULL; }
-int32_t syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype) { return 0; }
-void    syncStop(tsync_h shandle) {}
-int32_t syncReconfig(tsync_h shandle, const SSyncCfg * cfg) { return 0; }
-int     syncGetNodesRole(tsync_h shandle, SNodesRole * cfg) { return 0; }
-void    syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code) {}
+int64_t syncStart(const SSyncInfo *info) { return NULL; }
+int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int qtype) { return 0; }
+void    syncStop(int64_t rid) {}
+int32_t syncReconfig(int64_t rid, const SSyncCfg * cfg) { return 0; }
+int     syncGetNodesRole(int64_t rid, SNodesRole * cfg) { return 0; }
+void    syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {}
 #endif
 
 char* vnodeStatus[] = {
@@ -67,8 +61,8 @@ int32_t vnodeInitResources() {
   vnodeInitWriteFp();
   vnodeInitReadFp();
 
-  tsDnodeVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true);
-  if (tsDnodeVnodesHash == NULL) {
+  tsVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true);
+  if (tsVnodesHash == NULL) {
     vError("failed to init vnode list");
     return TSDB_CODE_VND_OUT_OF_MEMORY;
   }
@@ -77,10 +71,10 @@ int32_t vnodeInitResources() {
 }
 
 void vnodeCleanupResources() {
-  if (tsDnodeVnodesHash != NULL) {
+  if (tsVnodesHash != NULL) {
     vDebug("vnode list is cleanup");
-    taosHashCleanup(tsDnodeVnodesHash);
-    tsDnodeVnodesHash = NULL;
+    taosHashCleanup(tsVnodesHash);
+    tsVnodesHash = NULL;
   }
 
   syncCleanUp();
@@ -233,6 +227,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
 
   pVnode->vgId     = vnode;
   pVnode->status   = TAOS_VN_STATUS_INIT;
+  pVnode->fversion = 0;
   pVnode->version  = 0;  
   pVnode->tsdbCfg.tsdbId = pVnode->vgId;
   pVnode->rootDir = strdup(rootDir);
@@ -294,6 +289,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
       vnodeCleanUp(pVnode);
       return terrno;
     } else {
+      pVnode->fversion = 0;
       pVnode->version = 0;
     }
   }
@@ -308,6 +304,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
 
   walRestore(pVnode->wal, pVnode, vnodeProcessWrite);
   if (pVnode->version == 0) {
+    pVnode->fversion = 0;
     pVnode->version = walGetVersion(pVnode->wal);
   }
 
@@ -331,7 +328,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
 #ifndef _SYNC
   pVnode->role = TAOS_SYNC_ROLE_MASTER;
 #else
-  if (pVnode->sync == NULL) {
+  if (pVnode->sync <= 0) {
     vError("vgId:%d, failed to open sync module, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica,
            tstrerror(terrno));
     vnodeCleanUp(pVnode);
@@ -349,7 +346,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
   pVnode->status = TAOS_VN_STATUS_READY;
   vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode);
 
-  taosHashPut(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *));
+  taosHashPut(tsVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *));
 
   return TSDB_CODE_SUCCESS;
 }
@@ -441,7 +438,7 @@ void vnodeRelease(void *pVnodeRaw) {
   tsem_destroy(&pVnode->sem);
   free(pVnode);
 
-  int32_t count = taosHashGetSize(tsDnodeVnodesHash);
+  int32_t count = taosHashGetSize(tsVnodesHash);
   vDebug("vgId:%d, vnode is destroyed, vnodes:%d", vgId, count);
 }
 
@@ -458,7 +455,7 @@ static void vnodeIncRef(void *ptNode) {
 }
 
 void *vnodeAcquire(int32_t vgId) {
-  SVnodeObj **ppVnode = taosHashGetCB(tsDnodeVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *));
+  SVnodeObj **ppVnode = taosHashGetCB(tsVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *));
 
   if (ppVnode == NULL || *ppVnode == NULL) {
     terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
@@ -497,7 +494,7 @@ static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) {
 }
 
 int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
-  SHashMutableIterator *pIter = taosHashCreateIter(tsDnodeVnodesHash);
+  SHashMutableIterator *pIter = taosHashCreateIter(tsVnodesHash);
   while (taosHashIterNext(pIter)) {
     SVnodeObj **pVnode = taosHashIterGet(pIter);
     if (pVnode == NULL) continue;
@@ -518,7 +515,7 @@ int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) {
 
 void vnodeBuildStatusMsg(void *param) {
   SStatusMsg *pStatus = param;
-  SHashMutableIterator *pIter = taosHashCreateIter(tsDnodeVnodesHash);
+  SHashMutableIterator *pIter = taosHashCreateIter(tsVnodesHash);
 
   while (taosHashIterNext(pIter)) {
     SVnodeObj **pVnode = taosHashIterGet(pIter);
@@ -547,7 +544,7 @@ void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes) {
 
 static void vnodeCleanUp(SVnodeObj *pVnode) {
   // remove from hash, so new messages wont be consumed
-  taosHashRemove(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t));
+  taosHashRemove(tsVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t));
 
   if (pVnode->status != TAOS_VN_STATUS_INIT) {
     // it may be in updateing or reset state, then it shall wait
@@ -561,9 +558,9 @@ static void vnodeCleanUp(SVnodeObj *pVnode) {
   }
 
   // stop replication module
-  if (pVnode->sync) {
-    void *sync = pVnode->sync;
-    pVnode->sync = NULL;
+  if (pVnode->sync > 0) {
+    int64_t sync = pVnode->sync;
+    pVnode->sync = -1;
     syncStop(sync);
   }
 

src/vnode/src/vnodeRead.c

@@ -15,13 +15,10 @@
 
 #define _DEFAULT_SOURCE
 #define _NON_BLOCKING_RETRIEVE  0
-
 #include "os.h"
-
 #include "tglobal.h"
 #include "taoserror.h"
 #include "taosmsg.h"
-#include "tcache.h"
 #include "query.h"
 #include "trpc.h"
 #include "tsdb.h"
@@ -29,9 +26,9 @@
 #include "vnodeInt.h"
 #include "tqueue.h"
 
-static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *pVnode, SVReadMsg *pReadMsg);
-static int32_t  vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg);
-static int32_t  vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg);
+static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *pVnode, SVReadMsg *pRead);
+static int32_t  vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead);
+static int32_t  vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead);
 static int32_t  vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId);
 
 void vnodeInitReadFp(void) {
@@ -44,16 +41,16 @@ void vnodeInitReadFp(void) {
 // still required, or there will be a deadlock, so we don’t do any check here, but put the check codes before the
 // request enters the queue
 //
-int32_t vnodeProcessRead(void *param, SVReadMsg *pReadMsg) {
+int32_t vnodeProcessRead(void *param, SVReadMsg *pRead) {
   SVnodeObj *pVnode = (SVnodeObj *)param;
-  int32_t    msgType = pReadMsg->msgType;
+  int32_t    msgType = pRead->msgType;
 
   if (vnodeProcessReadMsgFp[msgType] == NULL) {
     vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[msgType]);
     return TSDB_CODE_VND_MSG_NOT_PROCESSED;
   }
 
-  return (*vnodeProcessReadMsgFp[msgType])(pVnode, pReadMsg);
+  return (*vnodeProcessReadMsgFp[msgType])(pVnode, pRead);
 }
 
 static int32_t vnodeCheckRead(void *param) {
@@ -180,27 +177,27 @@ static void vnodeBuildNoResultQueryRsp(SRspRet *pRet) {
   pRsp->completed = true;
 }
 
-static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
-  void *   pCont = pReadMsg->pCont;
-  int32_t  contLen = pReadMsg->contLen;
-  SRspRet *pRet = &pReadMsg->rspRet;
+static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
+  void *   pCont = pRead->pCont;
+  int32_t  contLen = pRead->contLen;
+  SRspRet *pRet = &pRead->rspRet;
 
   SQueryTableMsg *pQueryTableMsg = (SQueryTableMsg *)pCont;
   memset(pRet, 0, sizeof(SRspRet));
 
   // qHandle needs to be freed correctly
-  if (pReadMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
-    SRetrieveTableMsg *killQueryMsg = (SRetrieveTableMsg *)pReadMsg->pCont;
+  if (pRead->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
+    SRetrieveTableMsg *killQueryMsg = (SRetrieveTableMsg *)pRead->pCont;
     killQueryMsg->free = htons(killQueryMsg->free);
     killQueryMsg->qhandle = htobe64(killQueryMsg->qhandle);
 
-    vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pReadMsg->rpcHandle);
-    assert(pReadMsg->contLen > 0 && killQueryMsg->free == 1);
+    vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pRead->rpcHandle);
+    assert(pRead->contLen > 0 && killQueryMsg->free == 1);
 
     void **qhandle = qAcquireQInfo(pVnode->qMgmt, (uint64_t)killQueryMsg->qhandle);
     if (qhandle == NULL || *qhandle == NULL) {
       vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void *)killQueryMsg->qhandle,
-            pReadMsg->rpcHandle);
+            pRead->rpcHandle);
     } else {
       assert(*qhandle == (void *)killQueryMsg->qhandle);
 
@@ -242,9 +239,9 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
       }
 
       if (handle != NULL &&
-          vnodeNotifyCurrentQhandle(pReadMsg->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
+          vnodeNotifyCurrentQhandle(pRead->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
         vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle,
-               pReadMsg->rpcHandle);
+               pRead->rpcHandle);
         pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
         qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
         return pRsp->code;
@@ -255,7 +252,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
 
     if (handle != NULL) {
       vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, create qhandle and returns to app", vgId, *handle);
-      code = vnodePutItemIntoReadQueue(pVnode, handle, pReadMsg->rpcHandle);
+      code = vnodePutItemIntoReadQueue(pVnode, handle, pRead->rpcHandle);
       if (code != TSDB_CODE_SUCCESS) {
         pRsp->code = code;
         qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
@@ -264,7 +261,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
     }
   } else {
     assert(pCont != NULL);
-    void **qhandle = (void **)pReadMsg->qhandle;
+    void **qhandle = (void **)pRead->qhandle;
 
     vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle);
 
@@ -276,14 +273,14 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
     // build query rsp, the retrieve request has reached here already
     if (buildRes) {
       // update the connection info according to the retrieve connection
-      pReadMsg->rpcHandle = qGetResultRetrieveMsg(*qhandle);
-      assert(pReadMsg->rpcHandle != NULL);
+      pRead->rpcHandle = qGetResultRetrieveMsg(*qhandle);
+      assert(pRead->rpcHandle != NULL);
 
       vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle,
-             pReadMsg->rpcHandle);
+             pRead->rpcHandle);
 
       // set the real rsp error code
-      pReadMsg->code = vnodeDumpQueryResult(&pRead->rspRet, pVnode, qhandle, &freehandle, pReadMsg->rpcHandle);
+      pRead->code = vnodeDumpQueryResult(&pRead->rspRet, pVnode, qhandle, &freehandle, pRead->rpcHandle);
 
       // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client
       code = TSDB_CODE_QRY_HAS_RSP;
@@ -308,16 +305,16 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
   return code;
 }
 
-static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
-  void *   pCont = pReadMsg->pCont;
-  SRspRet *pRet = &pReadMsg->rspRet;
+static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
+  void *   pCont = pRead->pCont;
+  SRspRet *pRet = &pRead->rspRet;
 
   SRetrieveTableMsg *pRetrieve = pCont;
   pRetrieve->free = htons(pRetrieve->free);
   pRetrieve->qhandle = htobe64(pRetrieve->qhandle);
 
   vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void *)pRetrieve->qhandle,
-         pRetrieve->free, pReadMsg->rpcHandle);
+         pRetrieve->free, pRead->rpcHandle);
 
   memset(pRet, 0, sizeof(SRspRet));
 
@@ -348,8 +345,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
   }
 
   // register the qhandle to connect to quit query immediate if connection is broken
-  if (vnodeNotifyCurrentQhandle(pReadMsg->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
-    vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcHandle);
+  if (vnodeNotifyCurrentQhandle(pRead->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
+    vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, pRead->rpcHandle);
     code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
     qKillQuery(*handle);
     qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true);
@@ -359,7 +356,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
   bool freeHandle = true;
   bool buildRes = false;
 
-  code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcHandle);
+  code = qRetrieveQueryResultInfo(*handle, &buildRes, pRead->rpcHandle);
   if (code != TSDB_CODE_SUCCESS) {
     // TODO handle malloc failure
     pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
@@ -370,7 +367,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
     assert(buildRes == true);
 #if _NON_BLOCKING_RETRIEVE
     if (!buildRes) {
-      assert(pReadMsg->rpcHandle != NULL);
+      assert(pRead->rpcHandle != NULL);
 
       qReleaseQInfo(pVnode->qMgmt, (void **)&handle, false);
       return TSDB_CODE_QRY_NOT_READY;
@@ -378,7 +375,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) {
 #endif
 
     // ahandle is the sqlObj pointer
-    code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pReadMsg->rpcHandle);
+    code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pRead->rpcHandle);
   }
 
   // If qhandle is not added into vread queue, the query should be completed already or paused with error.

src/vnode/src/vnodeVersion.c

@@ -61,10 +61,10 @@ int32_t vnodeReadVersion(SVnodeObj *pVnode) {
     vError("vgId:%d, failed to read %s, version not found", pVnode->vgId, file);
     goto PARSE_VER_ERROR;
   }
-  pVnode->version = ver->valueint;
+  pVnode->version = (uint64_t)ver->valueint;
 
   terrno = TSDB_CODE_SUCCESS;
-  vInfo("vgId:%d, read %s successfully, version:%" PRId64, pVnode->vgId, file, pVnode->version);
+  vInfo("vgId:%d, read %s successfully, version:%" PRIu64, pVnode->vgId, file, pVnode->version);
 
 PARSE_VER_ERROR:
   if (content != NULL) free(content);
@@ -89,7 +89,7 @@ int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
   char *  content = calloc(1, maxLen + 1);
 
   len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"version\": %" PRId64 "\n", pVnode->fversion);
+  len += snprintf(content + len, maxLen - len, "  \"version\": %" PRIu64 "\n", pVnode->fversion);
   len += snprintf(content + len, maxLen - len, "}\n");
 
   fwrite(content, 1, len, fp);
@@ -98,6 +98,6 @@ int32_t vnodeSaveVersion(SVnodeObj *pVnode) {
   free(content);
   terrno = 0;
 
-  vInfo("vgId:%d, successed to write %s, version:%" PRId64, pVnode->vgId, file, pVnode->fversion);
+  vInfo("vgId:%d, successed to write %s, version:%" PRIu64, pVnode->vgId, file, pVnode->fversion);
   return TSDB_CODE_SUCCESS;
 }
\ No newline at end of file

src/vnode/src/vnodeWrite.c

@@ -19,7 +19,6 @@
 #include "taoserror.h"
 #include "tqueue.h"
 #include "trpc.h"
-#include "tutil.h"
 #include "tsdb.h"
 #include "twal.h"
 #include "tsync.h"
@@ -185,7 +184,7 @@ static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet
 
 static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) {
   SDropSTableMsg *pTable = pCont;
-  int32_t           code = TSDB_CODE_SUCCESS;
+  int32_t         code = TSDB_CODE_SUCCESS;
 
   vDebug("vgId:%d, stable:%s, start to drop", pVnode->vgId, pTable->tableId);
 

src/wal/inc/walInt.h

@@ -43,6 +43,7 @@ extern int32_t wDebugFlag;
 typedef struct {
   uint64_t version;
   int64_t  fileId;
+  int64_t  rid;
   int32_t  vgId;
   int32_t  fd;
   int32_t  keep;

src/wal/src/walMgmt.c

@@ -78,7 +78,8 @@ void *walOpen(char *path, SWalCfg *pCfg) {
     return NULL;
   }
 
-  if (taosAddRef(tsWal.refId, pWal) != TSDB_CODE_SUCCESS) {
+   pWal->rid = taosAddRef(tsWal.refId, pWal);
+   if (pWal->rid < 0) {
     walFreeObj(pWal);
     return NULL;
   }
@@ -127,7 +128,7 @@ void walClose(void *handle) {
 
   taosClose(pWal->fd);
 
-  if (!pWal->keep) {
+  if (pWal->keep != TAOS_WAL_KEEP) {
     int64_t fileId = -1;
     while (walGetNextFile(pWal, &fileId) >= 0) {
       snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId);
@@ -143,7 +144,7 @@ void walClose(void *handle) {
   }
 
   pthread_mutex_unlock(&pWal->mutex);
-  taosRemoveRef(tsWal.refId, pWal);
+  taosRemoveRef(tsWal.refId, pWal->rid);
 }
 
 static int32_t walInitObj(SWal *pWal) {
@@ -185,7 +186,7 @@ static void walUpdateSeq() {
 }
 
 static void walFsyncAll() {
-  SWal *pWal = taosIterateRef(tsWal.refId, NULL);
+  SWal *pWal = taosIterateRef(tsWal.refId, 0);
   while (pWal) {
     if (walNeedFsync(pWal)) {
       wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->vgId, pWal->level, pWal->fsyncSeq, tsWal.seq);
@@ -194,7 +195,7 @@ static void walFsyncAll() {
         wError("vgId:%d, file:%s, failed to fsync since %s", pWal->vgId, pWal->name, strerror(code));
       }
     }
-    pWal = taosIterateRef(tsWal.refId, pWal);
+    pWal = taosIterateRef(tsWal.refId, pWal->rid);
   }
 }
 

src/wal/src/walWrite.c

@@ -41,7 +41,7 @@ int32_t walRenew(void *handle) {
     wDebug("vgId:%d, file:%s, it is closed", pWal->vgId, pWal->name);
   }
 
-  if (pWal->keep) {
+  if (pWal->keep == TAOS_WAL_KEEP) {
     pWal->fileId = 0;
   } else {
     if (walGetNewFile(pWal, &pWal->fileId) != 0) pWal->fileId = 0;
@@ -58,7 +58,7 @@ int32_t walRenew(void *handle) {
     wDebug("vgId:%d, file:%s, it is created", pWal->vgId, pWal->name);
   }
 
-  if (!pWal->keep) {
+  if (pWal->keep != TAOS_WAL_KEEP) {
     // remove the oldest wal file
     int64_t oldFileId = -1;
     if (walGetOldFile(pWal, pWal->fileId, WAL_FILE_NUM, &oldFileId) == 0) {
@@ -144,12 +144,12 @@ int32_t walRestore(void *handle, void *pVnode, FWalWrite writeFp) {
       continue;
     }
 
-    wDebug("vgId:%d, file:%s, restore success and keep it", pWal->vgId, walName);
+    wDebug("vgId:%d, file:%s, restore success", pWal->vgId, walName);
 
     count++;
   }
 
-  if (!pWal->keep) return TSDB_CODE_SUCCESS;
+  if (pWal->keep != TAOS_WAL_KEEP) return TSDB_CODE_SUCCESS;
 
   if (count == 0) {
     wDebug("vgId:%d, wal file not exist, renew it", pWal->vgId);
@@ -173,7 +173,6 @@ int32_t walGetWalFile(void *handle, char *fileName, int64_t *fileId) {
   if (handle == NULL) return -1;
   SWal *pWal = handle;
 
-  // for keep
   if (*fileId == 0) *fileId = -1;
 
   pthread_mutex_lock(&(pWal->mutex));
@@ -311,7 +310,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch
   return code;
 }
 
-int64_t walGetVersion(twalh param) {
+uint64_t walGetVersion(twalh param) {
   SWal *pWal = param;
   if (pWal == 0) return 0;
 

src/wal/test/waltest.c

@@ -37,7 +37,6 @@ int writeToQueue(void *pVnode, void *data, int type, void *pMsg) {
 
 int main(int argc, char *argv[]) {
   char path[128] = "/home/jhtao/test/wal";
-  int  max = 3;
   int  level = 2;
   int  total = 5;
   int  rows = 10000;
@@ -47,8 +46,6 @@ int main(int argc, char *argv[]) {
   for (int i=1; i<argc; ++i) {
     if (strcmp(argv[i], "-p")==0 && i < argc-1) {
       tstrncpy(path, argv[++i], sizeof(path));
-    } else if (strcmp(argv[i], "-m")==0 && i < argc-1) {
-      max = atoi(argv[++i]);
     } else if (strcmp(argv[i], "-l")==0 && i < argc-1) {
       level = atoi(argv[++i]);
     } else if (strcmp(argv[i], "-r")==0 && i < argc-1) {
@@ -66,7 +63,6 @@ int main(int argc, char *argv[]) {
     } else {
       printf("\nusage: %s [options] \n", argv[0]);
       printf("  [-p path]: wal file path default is:%s\n", path);
-      printf("  [-m max]: max wal files, default is:%d\n", max);
       printf("  [-l level]: log level, default is:%d\n", level);
       printf("  [-t total]: total wal files, default is:%d\n", total);
       printf("  [-r rows]: rows of records per wal file, default is:%d\n", rows);
@@ -82,7 +78,6 @@ int main(int argc, char *argv[]) {
 
   SWalCfg walCfg;
   walCfg.walLevel = level;
-  walCfg.wals = max;
   walCfg.keep = keep;
 
   pWal = walOpen(path, &walCfg);

tests/perftest-scripts/perftest-query.sh

+#!/bin/bash
+
+today=`date +"%Y%m%d"`
+WORK_DIR=/home/ubuntu/pxiao/
+PERFORMANCE_TEST_REPORT=$TDENGINE_DIR/tests/performance-test-report-$today.log
+
+# Coloured Echoes                                                                                                       #
+function red_echo      { echo -e "\033[31m$@\033[0m";   }                                                               #
+function green_echo    { echo -e "\033[32m$@\033[0m";   }                                                               #
+function yellow_echo   { echo -e "\033[33m$@\033[0m";   }                                                               #
+function white_echo    { echo -e "\033[1;37m$@\033[0m"; }                                                               #
+# Coloured Printfs                                                                                                      #
+function red_printf    { printf "\033[31m$@\033[0m";    }                                                               #
+function green_printf  { printf "\033[32m$@\033[0m";    }                                                               #
+function yellow_printf { printf "\033[33m$@\033[0m";    }                                                               #
+function white_printf  { printf "\033[1;37m$@\033[0m";  }                                                               #
+# Debugging Outputs                                                                                                     #
+function white_brackets { local args="$@"; white_printf "["; printf "${args}"; white_printf "]"; }                      #
+function echoInfo   { local args="$@"; white_brackets $(green_printf "INFO") && echo " ${args}"; }                      #
+function echoWarn   { local args="$@";  echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; }  #
+function echoError  { local args="$@"; echo "$(white_brackets "$(red_printf    "ERROR")" && echo " ${args}";)" 1>&2; }  #
+
+
+function stopTaosd {
+	echo "Stop taosd"
+    systemctl stop taosd
+	snap stop tdengine
+  	PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+	while [ -n "$PID" ]
+	do
+    pkill -TERM -x taosd
+    sleep 1
+  	PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
+	done
+}
+
+function buildTDengine {
+	echoInfo "Build TDengine"
+	cd $WORK_DIR/TDengine
+
+	git remote update > /dev/null
+	REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
+	LOCAL_COMMIT=`git rev-parse --short @`
+
+	echo " LOCAL: $LOCAL_COMMIT"
+	echo "REMOTE: $REMOTE_COMMIT"
+	if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
+		echo "repo up-to-date"
+	else
+		echo "repo need to pull"
+		git pull > /dev/null
+
+		LOCAL_COMMIT=`git rev-parse --short @`
+		cd debug
+		rm -rf *
+		cmake .. > /dev/null
+		make > /dev/null
+		make install
+	fi
+}
+
+function runQueryPerfTest {
+	nohup $WORK_DIR/TDengine/debug/build/bin/taosd -c /etc/taodperf/ > /dev/null 2>&1 &
+	echoInfo "Run Performance Test"
+	cd $WORK_DIR/TDengine/tests/pytest
+	
+	python3 query/queryPerformance.py | tee -a $PERFORMANCE_TEST_REPORT
+}
+
+
+function sendReport {
+	echo "send report"
+	receiver="pxiao@taosdata.com"
+	mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n"
+
+	cd $TDENGINE_DIR
+
+	sed -i 's/\x1b\[[0-9;]*m//g' $PERFORMANCE_TEST_REPORT
+	BODY_CONTENT=`cat $PERFORMANCE_TEST_REPORT`
+	echo -e "to: ${receiver}\nsubject: Query Performace Report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
+	(cat - && uuencode $PERFORMANCE_TEST_REPORT performance-test-report-$today.log) | \
+	ssmtp "${receiver}" && echo "Report Sent!"
+}
+
+
+stopTaosd
+buildTDengine
+runQueryPerfTest
+
+echoInfo "Send Report"
+sendReport
+echoInfo "End of Test"

tests/pytest/crash_gen/crash_gen_main.py

@@ -2043,6 +2043,7 @@ class ThreadStacks: # stack info for all threads
                 print("[{sf}] File {filename}, line {lineno}, in {name}".format(
                     sf=stackFrame, filename=frame.filename, lineno=frame.lineno, name=frame.name))
                 print("    {}".format(frame.line))
+                stackFrame += 1
             print("-----> End of Thread Info ----->\n")
 
 class ClientManager:

tests/pytest/fulltest.sh

@@ -161,7 +161,7 @@ python3 ./test.py -f stream/metric_1.py
 python3 ./test.py -f stream/new.py
 python3 ./test.py -f stream/stream1.py
 python3 ./test.py -f stream/stream2.py
-python3 ./test.py -f stream/parser.py
+#python3 ./test.py -f stream/parser.py
 python3 ./test.py -f stream/history.py
 
 #alter table
@@ -207,3 +207,20 @@ python3 test.py -f tools/taosdemo.py
 python3 test.py -f subscribe/singlemeter.py
 #python3 test.py -f subscribe/stability.py
 python3 test.py -f subscribe/supertable.py
+
+
+# update
+python3 ./test.py -f update/allow_update.py
+python3 ./test.py -f update/allow_update-0.py
+python3 ./test.py -f update/append_commit_data.py
+python3 ./test.py -f update/append_commit_last-0.py
+python3 ./test.py -f update/append_commit_last.py
+python3 ./test.py -f update/merge_commit_data.py
+python3 ./test.py -f update/merge_commit_data-0.py
+python3 ./test.py -f update/merge_commit_data2.py
+python3 ./test.py -f update/merge_commit_data2_update0.py
+python3 ./test.py -f update/merge_commit_last-0.py
+python3 ./test.py -f update/merge_commit_last.py
+
+# wal
+python3 ./test.py -f wal/addOldWalTest.py
\ No newline at end of file