Merge remote-tracking branch 'origin/feature/scheduler' into feature/qnode

include/common/tmsg.h

@@ -1132,7 +1132,6 @@ int32_t tDeserializeSSchedulerHbReq(void* buf, int32_t bufLen, SSchedulerHbReq*
 void    tFreeSSchedulerHbReq(SSchedulerHbReq* pReq);
 
 typedef struct {
-  uint64_t       seqId;
   SQueryNodeEpId epId;
   SArray*        taskStatus;  // SArray<STaskStatus>
 } SSchedulerHbRsp;

include/common/tmsgdef.h

@@ -207,6 +207,10 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_SND_TASK_DEPLOY, "snode-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp)
   TD_DEF_MSG_TYPE(TDMT_SND_TASK_EXEC, "snode-task-exec", SStreamTaskExecReq, SStreamTaskExecRsp)
 
+  // Requests handled by SCHEDULER
+  TD_NEW_MSG_SEG(TDMT_SCH_MSG)
+  TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "scheduler-link-broken", NULL, NULL)
+  
 #if defined(TD_MSG_NUMBER_)
   TDMT_MAX
 #endif

include/libs/catalog/catalog.h

@@ -112,7 +112,7 @@ int32_t catalogUpdateDBVgInfo(SCatalog* pCatalog, const char* dbName, uint64_t d
 
 int32_t catalogRemoveDB(SCatalog* pCatalog, const char* dbName, uint64_t dbId);
 
-int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName);
+int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName);
 
 int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* stbName, uint64_t suid);
 

include/libs/nodes/cmdnodes.h

@@ -199,6 +199,7 @@ typedef struct SIndexOptions {
 typedef struct SCreateIndexStmt {
   ENodeType type;
   EIndexType indexType;
+  bool ignoreExists;
   char indexName[TSDB_INDEX_NAME_LEN];
   char tableName[TSDB_TABLE_NAME_LEN];
   SNodeList* pCols;
@@ -207,6 +208,7 @@ typedef struct SCreateIndexStmt {
 
 typedef struct SDropIndexStmt {
   ENodeType type;
+  bool ignoreNotExists;
   char indexName[TSDB_INDEX_NAME_LEN];
   char tableName[TSDB_TABLE_NAME_LEN];
 } SDropIndexStmt;

include/libs/qcom/query.h

@@ -150,6 +150,8 @@ int32_t cleanupTaskQueue();
  */
 int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code);
 
+int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo, bool persistHandle, void *ctx);
+
 /**
  * Asynchronously send message to server, after the response received, the callback will be incured.
  *

include/libs/transport/trpc.h

@@ -52,8 +52,8 @@ typedef struct {
   char    user[TSDB_USER_LEN];
   SRpcMsg rpcMsg;
   int32_t rspLen;
-  void   *pRsp;
-  void   *pNode;
+  void *  pRsp;
+  void *  pNode;
 } SNodeMsg;
 
 typedef struct SRpcInit {
@@ -81,13 +81,21 @@ typedef struct SRpcInit {
 } SRpcInit;
 
 typedef struct {
-  void *  val;
-  int32_t len;
-  void (*free)(void *arg);
+  void     *val;
+  int32_t (*clone)(void *src, void **dst);
+  void    (*free)(void *arg);
 } SRpcCtxVal;
 
 typedef struct {
-  SHashObj *args;
+  int32_t   msgType;
+  void     *val;
+  int32_t (*clone)(void *src, void **dst);
+  void    (*free)(void *arg);
+} SRpcBrokenlinkVal;
+
+typedef struct {
+  SHashObj *        args;
+  SRpcBrokenlinkVal brokenVal;
 } SRpcCtx;
 
 int32_t rpcInit();

source/client/src/clientEnv.c

@@ -192,6 +192,10 @@ static void doDestroyRequest(void *p) {
   doFreeReqResultInfo(&pRequest->body.resInfo);
   qDestroyQueryPlan(pRequest->body.pDag);
 
+  if (pRequest->body.queryJob != 0) {
+    schedulerFreeJob(pRequest->body.queryJob);
+  }
+
   if (pRequest->body.showInfo.pArray != NULL) {
     taosArrayDestroy(pRequest->body.showInfo.pArray);
   }

source/client/src/clientImpl.c

@@ -330,6 +330,8 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
       pRequest->code = code;
       break;
     }
+
+    destroyRequest(pRequest);
   }
   
   return pRequest;

source/common/src/tglobal.c

@@ -180,6 +180,10 @@ static int32_t taosSetTfsCfg(SConfig *pCfg) {
       memcpy(&tsDiskCfg[index], pCfg, sizeof(SDiskCfg));
       if (pCfg->level == 0 && pCfg->primary == 1) {
         tstrncpy(tsDataDir, pCfg->dir, PATH_MAX);
+        if (taosMkDir(tsDataDir) != 0) {
+          uError("failed to create dataDir:%s since %s", tsDataDir, terrstr());
+          return -1;
+        }
       }
       if (taosMkDir(pCfg->dir) != 0) {
         uError("failed to create tfsDir:%s since %s", tsDataDir, terrstr());

source/common/src/tmsg.c

@@ -2871,7 +2871,6 @@ int32_t tSerializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *pR
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
 
   if (tStartEncode(&encoder) < 0) return -1;
-  if (tEncodeU64(&encoder, pRsp->seqId) < 0) return -1;
   if (tEncodeI32(&encoder, pRsp->epId.nodeId) < 0) return -1;
   if (tEncodeU16(&encoder, pRsp->epId.ep.port) < 0) return -1;
   if (tEncodeCStr(&encoder, pRsp->epId.ep.fqdn) < 0) return -1;
@@ -2900,7 +2899,6 @@ int32_t tDeserializeSSchedulerHbRsp(void *buf, int32_t bufLen, SSchedulerHbRsp *
   tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
 
   if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeU64(&decoder, &pRsp->seqId) < 0) return -1;
   if (tDecodeI32(&decoder, &pRsp->epId.nodeId) < 0) return -1;
   if (tDecodeU16(&decoder, &pRsp->epId.ep.port) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pRsp->epId.ep.fqdn) < 0) return -1;

source/dnode/mgmt/bnode/src/bmWorker.c

@@ -77,7 +77,11 @@ int32_t bmStartWorker(SBnodeMgmt *pMgmt) {
     return -1;
   }
 
+  dDebug("bnode workers are initialized");
   return 0;
 }
 
-void bmStopWorker(SBnodeMgmt *pMgmt) { tMultiWorkerCleanup(&pMgmt->writeWorker); }
+void bmStopWorker(SBnodeMgmt *pMgmt) {
+  tMultiWorkerCleanup(&pMgmt->writeWorker);
+  dDebug("bnode workers are closed");
+}

source/dnode/mgmt/container/inc/dnd.h

@@ -19,7 +19,6 @@
 #include "os.h"
 
 #include "cJSON.h"
-#include "monitor.h"
 #include "tcache.h"
 #include "tcrc32c.h"
 #include "tdatablock.h"
@@ -36,8 +35,7 @@
 #include "tworker.h"
 
 #include "dnode.h"
-#include "tfs.h"
-#include "wal.h"
+#include "monitor.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -53,7 +51,6 @@ extern "C" {
 typedef enum { DNODE, VNODES, QNODE, SNODE, MNODE, BNODE, NODE_MAX } ENodeType;
 typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EDndStatus;
 typedef enum { DND_ENV_INIT, DND_ENV_READY, DND_ENV_CLEANUP } EEnvStatus;
-typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType;
 typedef enum { PROC_SINGLE, PROC_CHILD, PROC_PARENT } EProcType;
 
 typedef struct SMgmtFp      SMgmtFp;
@@ -127,29 +124,30 @@ typedef struct SDnode {
   bool         dropped;
   EDndStatus   status;
   EDndEvent    event;
-  EProcType    procType;
   SStartupReq  startup;
   TdFilePtr    pLockFile;
   STransMgmt   trans;
   SMgmtWrapper wrappers[NODE_MAX];
 } SDnode;
 
-EDndStatus    dndGetStatus(SDnode *pDnode);
-void          dndSetStatus(SDnode *pDnode, EDndStatus stat);
-SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType);
-void          dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp, int32_t vgId);
-void          dndReportStartup(SDnode *pDnode, char *pName, char *pDesc);
-void          dndSendMonitorReport(SDnode *pDnode);
+EDndStatus dndGetStatus(SDnode *pDnode);
+void       dndSetStatus(SDnode *pDnode, EDndStatus stat);
+void       dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp, int32_t vgId);
+void       dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc);
+void       dndSendMonitorReport(SDnode *pDnode);
 
 int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
 int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pMsg);
 void    dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp);
 
 int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg);
-
 int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed);
 int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed);
 
+SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType);
+int32_t       dndMarkWrapper(SMgmtWrapper *pWrapper);
+void          dndReleaseWrapper(SMgmtWrapper *pWrapper);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mgmt/container/inc/dndInt.h

@@ -34,7 +34,7 @@ int32_t     dndInit();
 void        dndCleanup();
 const char *dndStatStr(EDndStatus stat);
 void        dndGetStartup(SDnode *pDnode, SStartupReq *pStartup);
-TdFilePtr   dndCheckRunning(char *dataDir);
+TdFilePtr   dndCheckRunning(const char *dataDir);
 void        dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg);
 
 // dndMsg.c
@@ -50,10 +50,6 @@ SDnode *dndCreate(const SDnodeOpt *pOption);
 void    dndClose(SDnode *pDnode);
 void    dndHandleEvent(SDnode *pDnode, EDndEvent event);
 
-SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType);
-int32_t       dndMarkWrapper(SMgmtWrapper *pWrapper);
-void          dndReleaseWrapper(SMgmtWrapper *pWrapper);
-
 // dndTransport.c
 int32_t dndInitServer(SDnode *pDnode);
 void    dndCleanupServer(SDnode *pDnode);

source/dnode/mgmt/container/src/dndExec.c

@@ -50,16 +50,23 @@ int32_t dndOpenNode(SMgmtWrapper *pWrapper) {
 }
 
 void dndCloseNode(SMgmtWrapper *pWrapper) {
+  dDebug("node:%s, start to close", pWrapper->name);
   taosWLockLatch(&pWrapper->latch);
   if (pWrapper->deployed) {
     (*pWrapper->fp.closeFp)(pWrapper);
     pWrapper->deployed = false;
   }
+  taosWUnLockLatch(&pWrapper->latch);
+
+  while (pWrapper->refCount > 0) {
+    taosMsleep(10);
+  }
+
   if (pWrapper->pProc) {
     taosProcCleanup(pWrapper->pProc);
     pWrapper->pProc = NULL;
   }
-  taosWUnLockLatch(&pWrapper->latch);
+  dDebug("node:%s, has been closed", pWrapper->name);
 }
 
 static int32_t dndRunInSingleProcess(SDnode *pDnode) {

source/dnode/mgmt/container/src/dndInt.c

@@ -15,11 +15,12 @@
 
 #define _DEFAULT_SOURCE
 #include "dndInt.h"
+#include "wal.h"
 
 static int8_t once = DND_ENV_INIT;
 
 int32_t dndInit() {
-  dDebug("start to init dnode env");
+  dInfo("start to init dnode env");
   if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) {
     terrno = TSDB_CODE_REPEAT_INIT;
     dError("failed to init dnode env since %s", terrstr());
@@ -52,7 +53,7 @@ int32_t dndInit() {
 }
 
 void dndCleanup() {
-  dDebug("start to cleanup dnode env");
+  dInfo("start to cleanup dnode env");
   if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) {
     dError("dnode env is already cleaned up");
     return;
@@ -92,7 +93,7 @@ const char *dndStatStr(EDndStatus status) {
   }
 }
 
-void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc) {
+void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) {
   SStartupReq *pStartup = &pDnode->startup;
   tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN);
   tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN);
@@ -104,21 +105,21 @@ void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) {
   pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING);
 }
 
-TdFilePtr dndCheckRunning(char *dataDir) {
+TdFilePtr dndCheckRunning(const char *dataDir) {
   char filepath[PATH_MAX] = {0};
   snprintf(filepath, sizeof(filepath), "%s/.running", dataDir);
 
   TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
-    dError("failed to lock file:%s since %s, quit", filepath, terrstr());
+    dError("failed to lock file:%s since %s", filepath, terrstr());
     return NULL;
   }
 
   int32_t ret = taosLockFile(pFile);
   if (ret != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
-    dError("failed to lock file:%s since %s, quit", filepath, terrstr());
+    dError("failed to lock file:%s since %s", filepath, terrstr());
     taosCloseFile(&pFile);
     return NULL;
   }
@@ -129,12 +130,11 @@ TdFilePtr dndCheckRunning(char *dataDir) {
 
 void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) {
   dDebug("startup req is received");
-
   SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq));
   dndGetStartup(pDnode, pStartup);
 
   dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished);
-
-  SRpcMsg rpcRsp = {.handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq)};
+  SRpcMsg rpcRsp = {
+      .handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq), .ahandle = pReq->ahandle};
   rpcSendResponse(&rpcRsp);
 }

source/dnode/mgmt/container/src/dndMonitor.c

@@ -22,7 +22,12 @@ static int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) {
   tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name));
   pInfo->tempdir.size = tsTempSpace.size;
 
-  return vmMonitorTfsInfo(dndAcquireWrapper(pDnode, VNODES), pInfo);
+  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES);
+  if (pWrapper != NULL) {
+    vmMonitorTfsInfo(pWrapper, pInfo);
+    dndReleaseWrapper(pWrapper);
+  }
+  return 0;
 }
 
 static void dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
@@ -45,8 +50,17 @@ static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
   taosGetCardInfo(&pInfo->net_in, &pInfo->net_out);
   taosGetProcIO(&pInfo->io_read, &pInfo->io_write, &pInfo->io_read_disk, &pInfo->io_write_disk);
 
-  vmMonitorVnodeReqs(dndAcquireWrapper(pDnode, VNODES), pInfo);
-  pInfo->has_mnode = (dndAcquireWrapper(pDnode, MNODE)->required);
+  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES);
+  if (pWrapper != NULL) {
+    vmMonitorVnodeReqs(pWrapper, pInfo);
+    dndReleaseWrapper(pWrapper);
+  }
+
+  pWrapper = dndAcquireWrapper(pDnode, MNODE);
+  if (pWrapper != NULL) {
+    pInfo->has_mnode = pWrapper->required;
+    dndReleaseWrapper(pWrapper);
+  }
 }
 
 void dndSendMonitorReport(SDnode *pDnode) {
@@ -63,10 +77,15 @@ void dndSendMonitorReport(SDnode *pDnode) {
   SMonClusterInfo clusterInfo = {0};
   SMonVgroupInfo  vgroupInfo = {0};
   SMonGrantInfo   grantInfo = {0};
-  if (mmMonitorMnodeInfo(dndAcquireWrapper(pDnode, MNODE), &clusterInfo, &vgroupInfo, &grantInfo) == 0) {
-    monSetClusterInfo(pMonitor, &clusterInfo);
-    monSetVgroupInfo(pMonitor, &vgroupInfo);
-    monSetGrantInfo(pMonitor, &grantInfo);
+
+  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, MNODE);
+  if (pWrapper != NULL) {
+    if (mmMonitorMnodeInfo(pWrapper, &clusterInfo, &vgroupInfo, &grantInfo) == 0) {
+      monSetClusterInfo(pMonitor, &clusterInfo);
+      monSetVgroupInfo(pMonitor, &vgroupInfo);
+      monSetGrantInfo(pMonitor, &grantInfo);
+    }
+    dndReleaseWrapper(pWrapper);
   }
 
   SMonDnodeInfo dnodeInfo = {0};

source/dnode/mgmt/container/src/dndMsg.c

@@ -20,8 +20,8 @@ static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) {
   SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE);
   if (pWrapper != NULL) {
     dmUpdateMnodeEpSet(pWrapper->pMgmt, pEpSet);
+    dndReleaseWrapper(pWrapper);
   }
-  dndReleaseWrapper(pWrapper);
 }
 
 static inline NodeMsgFp dndGetMsgFp(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {

source/dnode/mgmt/container/src/dndObj.c

@@ -58,7 +58,7 @@ static void dndClearMemory(SDnode *pDnode) {
 SDnode *dndCreate(const SDnodeOpt *pOption) {
   dInfo("start to create dnode object");
   int32_t code = -1;
-  char    path[PATH_MAX];
+  char    path[PATH_MAX] = {0};
   SDnode *pDnode = NULL;
 
   pDnode = calloc(1, sizeof(SDnode));

source/dnode/mgmt/container/src/dndTransport.c

@@ -146,9 +146,14 @@ static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) {
 
 static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRpcRsp) {
   STransMgmt *pMgmt = &pDnode->trans;
+  SEpSet      epSet = {0};
+
+  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE);
+  if (pWrapper != NULL) {
+    dmGetMnodeEpSet(pWrapper->pMgmt, &epSet);
+    dndReleaseWrapper(pWrapper);
+  }
 
-  SEpSet epSet = {0};
-  dmGetMnodeEpSet(dndAcquireWrapper(pDnode, DNODE)->pMgmt, &epSet);
   rpcSendRecv(pMgmt->clientRpc, &epSet, pRpcMsg, pRpcRsp);
 }
 
@@ -182,9 +187,14 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char
     return 0;
   }
 
-  if (mmGetUserAuth(dndAcquireWrapper(pDnode, MNODE), user, spi, encrypt, secret, ckey) == 0) {
-    dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt);
-    return 0;
+  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, MNODE);
+  if (pWrapper != NULL) {
+    if (mmGetUserAuth(pWrapper, user, spi, encrypt, secret, ckey) == 0) {
+      dndReleaseWrapper(pWrapper);
+      dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt);
+      return 0;
+    }
+    dndReleaseWrapper(pWrapper);
   }
 
   if (terrno != TSDB_CODE_APP_NOT_READY) {
@@ -271,7 +281,7 @@ int32_t dndInitMsgHandle(SDnode *pDnode) {
       int32_t   vgId = pWrapper->msgVgIds[msgIndex];
       if (msgFp == NULL) continue;
 
-      dTrace("msg:%s will be processed by %s, vgId:%d", tMsgInfo[msgIndex], pWrapper->name, vgId);
+      // dTrace("msg:%s will be processed by %s, vgId:%d", tMsgInfo[msgIndex], pWrapper->name, vgId);
 
       SMsgHandle *pHandle = &pMgmt->msgHandles[msgIndex];
       if (vgId == QND_VGID) {
@@ -328,7 +338,12 @@ int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pReq) {
     SDnode     *pDnode = pWrapper->pDnode;
     STransMgmt *pTrans = &pDnode->trans;
     SEpSet      epSet = {0};
-    dmGetMnodeEpSet(dndAcquireWrapper(pDnode, DNODE)->pMgmt, &epSet);
+
+    SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE);
+    if (pWrapper != NULL) {
+      dmGetMnodeEpSet(pWrapper->pMgmt, &epSet);
+      dndReleaseWrapper(pWrapper);
+    }
     return dndSendRpcReq(pTrans, &epSet, pReq);
   }
 }
@@ -336,7 +351,12 @@ int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pReq) {
 void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) {
   if (pRsp->code == TSDB_CODE_APP_NOT_READY) {
     SMgmtWrapper *pDnodeWrapper = dndAcquireWrapper(pWrapper->pDnode, DNODE);
-    dmSendRedirectRsp(pDnodeWrapper->pMgmt, pRsp);
+    if (pDnodeWrapper != NULL) {
+      dmSendRedirectRsp(pDnodeWrapper->pMgmt, pRsp);
+      dndReleaseWrapper(pDnodeWrapper);
+    } else {
+      rpcSendResponse(pRsp);
+    }
   } else {
     rpcSendResponse(pRsp);
   }

source/dnode/mgmt/dnode/src/dmFile.c

@@ -209,7 +209,7 @@ int32_t dmWriteFile(SDnodeMgmt *pMgmt) {
   }
 
   pMgmt->updateTime = taosGetTimestampMs();
-  dDebug("successed to write %s", file);
+  dDebug("successed to write %s", realfile);
   return 0;
 }
 

source/dnode/mgmt/dnode/src/dmInt.c

@@ -74,14 +74,14 @@ void dmSendRedirectRsp(SDnodeMgmt *pMgmt, SRpcMsg *pReq) {
 }
 
 static int32_t dmStart(SMgmtWrapper *pWrapper) {
-  dDebug("dnode mgmt start to run");
+  dDebug("dnode-mgmt start to run");
   return dmStartThread(pWrapper->pMgmt);
 }
 
 int32_t dmInit(SMgmtWrapper *pWrapper) {
   SDnode     *pDnode = pWrapper->pDnode;
   SDnodeMgmt *pMgmt = calloc(1, sizeof(SDnodeMgmt));
-  dInfo("dnode-mgmt is initialized");
+  dInfo("dnode-mgmt start to init");
 
   pDnode->dnodeId = 0;
   pDnode->dropped = 0;

source/dnode/mgmt/dnode/src/dmMsg.c

@@ -41,8 +41,12 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
   memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN);
   taosRUnLockLatch(&pMgmt->latch);
 
-  req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad));
-  vmMonitorVnodeLoads(dndAcquireWrapper(pDnode, VNODES), req.pVloads);
+  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, VNODES);
+  if (pWrapper != NULL) {
+    req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad));
+    vmMonitorVnodeLoads(pWrapper, req.pVloads);
+    dndReleaseWrapper(pWrapper);
+  }
 
   int32_t contLen = tSerializeSStatusReq(NULL, 0, &req);
   void   *pHead = rpcMallocCont(contLen);

source/dnode/mgmt/dnode/src/dmWorker.c

@@ -23,7 +23,7 @@
 
 static void *dmThreadRoutine(void *param) {
   SDnodeMgmt *pMgmt = param;
-  SDnode     *pDnode = pMgmt->pDnode;
+  SDnode *    pDnode = pMgmt->pDnode;
   int64_t     lastStatusTime = taosGetTimestampMs();
   int64_t     lastMonitorTime = lastStatusTime;
 
@@ -55,7 +55,7 @@ static void *dmThreadRoutine(void *param) {
 static void dmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
   SDnodeMgmt *pMgmt = pInfo->ahandle;
 
-  SDnode  *pDnode = pMgmt->pDnode;
+  SDnode * pDnode = pMgmt->pDnode;
   SRpcMsg *pRpc = &pMsg->rpcMsg;
   int32_t  code = -1;
   dTrace("msg:%p, will be processed in dnode queue", pMsg);
@@ -114,6 +114,7 @@ int32_t dmStartWorker(SDnodeMgmt *pMgmt) {
     return -1;
   }
 
+  dDebug("dnode workers are initialized");
   return 0;
 }
 
@@ -136,6 +137,7 @@ void dmStopWorker(SDnodeMgmt *pMgmt) {
     taosDestoryThread(pMgmt->threadId);
     pMgmt->threadId = NULL;
   }
+  dDebug("dnode workers are closed");
 }
 
 int32_t dmProcessMgmtMsg(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
@@ -144,6 +146,6 @@ int32_t dmProcessMgmtMsg(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
     pWorker = &pMgmt->statusWorker;
   }
 
-  dTrace("msg:%p, will be written to worker %s", pMsg, pWorker->name);
+  dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
   return taosWriteQitem(pWorker->queue, pMsg);
 }

source/dnode/mgmt/mnode/src/mmInt.c

@@ -15,6 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "mmInt.h"
+#include "wal.h"
 
 static bool mmDeployRequired(SDnode *pDnode) {
   if (pDnode->dnodeId > 0) return false;
@@ -226,7 +227,7 @@ static int32_t mmOpen(SMgmtWrapper *pWrapper) {
 }
 
 static int32_t mmStart(SMgmtWrapper *pWrapper) {
-  dDebug("mnode mgmt start to run");
+  dDebug("mnode-mgmt start to run");
   SMnodeMgmt *pMgmt = pWrapper->pMgmt;
   return mndStart(pMgmt->pMnode);
 }

source/dnode/mgmt/mnode/src/mmWorker.c

@@ -108,6 +108,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) {
     return -1;
   }
 
+  dDebug("mnode workers are initialized");
   return 0;
 }
 
@@ -115,4 +116,5 @@ void mmStopWorker(SMnodeMgmt *pMgmt) {
   tSingleWorkerCleanup(&pMgmt->readWorker);
   tSingleWorkerCleanup(&pMgmt->writeWorker);
   tSingleWorkerCleanup(&pMgmt->syncWorker);
+  dDebug("mnode workers are closed");
 }

source/dnode/mgmt/qnode/src/qmWorker.c

@@ -132,10 +132,12 @@ int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
     return -1;
   }
 
+  dDebug("qnode workers are initialized");
   return 0;
 }
 
 void qmStopWorker(SQnodeMgmt *pMgmt) {
   tSingleWorkerCleanup(&pMgmt->queryWorker);
   tSingleWorkerCleanup(&pMgmt->fetchWorker);
+  dDebug("qnode workers are closed");
 }

source/dnode/mgmt/snode/src/smWorker.c

@@ -80,6 +80,7 @@ int32_t smStartWorker(SSnodeMgmt *pMgmt) {
     return -1;
   }
 
+  dDebug("snode workers are initialized");
   return 0;
 }
 
@@ -90,6 +91,7 @@ void smStopWorker(SSnodeMgmt *pMgmt) {
   }
   taosArrayDestroy(pMgmt->uniqueWorkers);
   tSingleWorkerCleanup(&pMgmt->sharedWorker);
+  dDebug("snode workers are closed");
 }
 
 static FORCE_INLINE int32_t smGetSWIdFromMsg(SRpcMsg *pMsg) {

source/dnode/mgmt/vnode/src/vmInt.c

@@ -257,14 +257,14 @@ static void vmCleanup(SMgmtWrapper *pWrapper) {
   SVnodesMgmt *pMgmt = pWrapper->pMgmt;
   if (pMgmt == NULL) return;
 
-  dInfo("vnodes-mgmt start to cleanup");
+  dInfo("vnode-mgmt start to cleanup");
   vmCloseVnodes(pMgmt);
   vmStopWorker(pMgmt);
   vnodeCleanup();
   // walCleanUp();
   free(pMgmt);
   pWrapper->pMgmt = NULL;
-  dInfo("vnodes-mgmt is cleaned up");
+  dInfo("vnode-mgmt is cleaned up");
 }
 
 static int32_t vmInit(SMgmtWrapper *pWrapper) {
@@ -272,7 +272,7 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) {
   SVnodesMgmt *pMgmt = calloc(1, sizeof(SVnodesMgmt));
   int32_t      code = -1;
 
-  dInfo("vnodes-mgmt start to init");
+  dInfo("vnode-mgmt start to init");
   if (pMgmt == NULL) goto _OVER;
 
   pMgmt->path = pWrapper->path;
@@ -312,7 +312,7 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) {
   }
 
   if (vmOpenVnodes(pMgmt) != 0) {
-    dError("failed to open vnodes since %s", terrstr());
+    dError("failed to open vnode since %s", terrstr());
     return -1;
   }
 

source/dnode/mgmt/vnode/src/vmWorker.c

@@ -165,8 +165,8 @@ static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueTyp
   int32_t  code = -1;
 
   SMsgHead *pHead = pRpc->pCont;
-  pHead->contLen = htonl(pHead->contLen);
-  pHead->vgId = htonl(pHead->vgId);
+  pHead->contLen = ntohl(pHead->contLen);
+  pHead->vgId = ntohl(pHead->vgId);
 
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
   if (pVnode == NULL) {
@@ -356,7 +356,7 @@ int32_t vmStartWorker(SVnodesMgmt *pMgmt) {
     return -1;
   }
 
-  dDebug("vnode workers is initialized");
+  dDebug("vnode workers are initialized");
   return 0;
 }
 
@@ -366,5 +366,5 @@ void vmStopWorker(SVnodesMgmt *pMgmt) {
   tQWorkerCleanup(&pMgmt->queryPool);
   tWWorkerCleanup(&pMgmt->writePool);
   tWWorkerCleanup(&pMgmt->syncPool);
-  dDebug("vnode workers is closed");
+  dDebug("vnode workers are closed");
 }

source/dnode/mnode/impl/src/mndDb.c

@@ -1034,7 +1034,9 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) {
       usedbRsp.vgVersion = usedbReq.vgVersion;
       code = 0;
     }
-    usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos);    
+    usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos);  
+
+    // no jump, need to construct rsp
   } else {
     pDb = mndAcquireDb(pMnode, usedbReq.db);
     if (pDb == NULL) {

source/dnode/vnode/src/vnd/vnodeQuery.c

@@ -31,9 +31,8 @@ int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
   SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta, .config = &pVnode->config};
 
   switch (pMsg->msgType) {
-    case TDMT_VND_QUERY: {
+    case TDMT_VND_QUERY:
       return qWorkerProcessQueryMsg(&handle, pVnode->pQuery, pMsg);
-    }
     case TDMT_VND_QUERY_CONTINUE:
       return qWorkerProcessCQueryMsg(&handle, pVnode->pQuery, pMsg);
     default:
@@ -205,7 +204,7 @@ _exit:
 
   rpcSendResponse(&rpcMsg);
 
-  return code;
+  return TSDB_CODE_SUCCESS;
 }
 
 static void freeItemHelper(void *pItem) {

source/libs/catalog/src/catalog.c

@@ -406,9 +406,9 @@ _return:
 }
 
 
-int32_t ctgPushRmStbMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid) {
+int32_t ctgPushRmStbMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncReq) {
   int32_t code = 0;
-  SCtgMetaAction action= {.act = CTG_ACT_REMOVE_STB};
+  SCtgMetaAction action= {.act = CTG_ACT_REMOVE_STB, .syncReq = syncReq};
   SCtgRemoveStbMsg *msg = malloc(sizeof(SCtgRemoveStbMsg));
   if (NULL == msg) {
     ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveStbMsg));
@@ -435,9 +435,9 @@ _return:
 
 
 
-int32_t ctgPushRmTblMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName) {
+int32_t ctgPushRmTblMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncReq) {
   int32_t code = 0;
-  SCtgMetaAction action= {.act = CTG_ACT_REMOVE_TBL};
+  SCtgMetaAction action= {.act = CTG_ACT_REMOVE_TBL, .syncReq = syncReq};
   SCtgRemoveTblMsg *msg = malloc(sizeof(SCtgRemoveTblMsg));
   if (NULL == msg) {
     ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveTblMsg));
@@ -496,7 +496,7 @@ _return:
 
 int32_t ctgPushUpdateTblMsgInQueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq) {
   int32_t code = 0;
-  SCtgMetaAction action= {.act = CTG_ACT_UPDATE_TBL};
+  SCtgMetaAction action= {.act = CTG_ACT_UPDATE_TBL, .syncReq = syncReq};
   SCtgUpdateTblMsg *msg = malloc(sizeof(SCtgUpdateTblMsg));
   if (NULL == msg) {
     ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg));
@@ -1843,6 +1843,7 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps,
 
   if (CTG_IS_META_NULL(output->metaType)) {
     ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(pTableName));
+    catalogRemoveTableMeta(pCtg, pTableName);
     CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST);
   }
 
@@ -1951,9 +1952,9 @@ _return:
     }
 
     if (TSDB_SUPER_TABLE == tbType) {
-      ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, suid);
+      ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, suid, false);
     } else {
-      ctgPushRmTblMsgInQueue(pCtg, dbFName, dbId, pTableName->tname);
+      ctgPushRmTblMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, false);
     }
   }
 
@@ -2534,7 +2535,7 @@ int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId,
 
 }
 
-int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName) {
+int32_t catalogRemoveTableMeta(SCatalog* pCtg, const SName* pTableName) {
   CTG_API_ENTER();
 
   int32_t code = 0;
@@ -2561,9 +2562,9 @@ int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName) {
   tNameGetFullDbName(pTableName, dbFName);
   
   if (TSDB_SUPER_TABLE == tblMeta->tableType) {
-    CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, tblMeta->suid));
+    CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, tblMeta->suid, true));
   } else {
-    CTG_ERR_JRET(ctgPushRmTblMsgInQueue(pCtg, dbFName, dbId, pTableName->tname));
+    CTG_ERR_JRET(ctgPushRmTblMsgInQueue(pCtg, dbFName, dbId, pTableName->tname, true));
   }
 
  
@@ -2588,7 +2589,7 @@ int32_t catalogRemoveStbMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId,
     CTG_API_LEAVE(TSDB_CODE_SUCCESS);
   }
 
-  CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, stbName, suid));
+  CTG_ERR_JRET(ctgPushRmStbMsgInQueue(pCtg, dbFName, dbId, stbName, suid, true));
 
   CTG_API_LEAVE(TSDB_CODE_SUCCESS);
   

source/libs/function/CMakeLists.txt

 aux_source_directory(src FUNCTION_SRC)
+list(REMOVE_ITEM FUNCTION_SRC src/udfd.c)
 add_library(function STATIC ${FUNCTION_SRC})
 target_include_directories(
     function
-    PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/function"
+    PUBLIC
+        "${CMAKE_SOURCE_DIR}/include/libs/function"
+        "${CMAKE_SOURCE_DIR}/contrib/libuv/include"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 
 target_link_libraries(
     function
+    PUBLIC uv_a
     PRIVATE os util common nodes
-)
+)
\ No newline at end of file
+
+add_executable(runUdf test/runUdf.c)
+target_include_directories(
+        runUdf
+        PUBLIC
+            "${CMAKE_SOURCE_DIR}/include/libs/function"
+            "${CMAKE_SOURCE_DIR}/contrib/libuv/include"
+        PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+)
+target_link_libraries(
+        runUdf
+        PUBLIC uv_a
+        PRIVATE os util common nodes function
+)
+
+add_library(udf1 MODULE test/udf1.c)
+target_include_directories(
+        udf1
+        PUBLIC
+            "${CMAKE_SOURCE_DIR}/include/libs/function"
+        PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+)
+
+#SET(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/build/bin)
+add_executable(udfd src/udfd.c)
+target_include_directories(
+        udfd
+        PUBLIC
+            "${CMAKE_SOURCE_DIR}/include/libs/function"
+            "${CMAKE_SOURCE_DIR}/contrib/libuv/include"
+        PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+)
+
+target_link_libraries(
+        udfd
+        PUBLIC uv_a
+        PRIVATE os util common nodes function
+)
+

source/libs/function/inc/tudf.h

@@ -20,68 +20,116 @@
 extern "C" {
 #endif
 
-#include "os.h"
-#include "taoserror.h"
+//======================================================================================
+//begin API to taosd and qworker
+/**
+ * start udf dameon service
+ * @return error code
+ */
+int32_t startUdfService();
+
+/**
+ * stop udf dameon service
+ * @return error code
+ */
+int32_t stopUdfService();
 
 enum {
-  TSDB_UDF_FUNC_NORMAL = 0,
-  TSDB_UDF_FUNC_INIT,
-  TSDB_UDF_FUNC_FINALIZE,
-  TSDB_UDF_FUNC_MERGE,
-  TSDB_UDF_FUNC_DESTROY,
-  TSDB_UDF_FUNC_MAX_NUM
+  TSDB_UDF_TYPE_SCALAR = 0,
+  TSDB_UDF_TYPE_AGGREGATE = 1
 };
 
-typedef struct SUdfInit {
-  int32_t  maybe_null; /* 1 if function can return NULL */
-  uint32_t decimals;   /* for real functions */
-  uint64_t length;     /* For string functions */
-  char*    ptr;        /* free pointer for function data */
-  int32_t  const_item; /* 0 if result is independent of arguments */
-
-  // script like lua/javascript
-  void* script_ctx;
-  void (*destroyCtxFunc)(void* script_ctx);
-} SUdfInit;
+enum {
+  TSDB_UDF_SCRIPT_BIN_LIB = 0,
+  TSDB_UDF_SCRIPT_LUA = 1,
+};
 
 typedef struct SUdfInfo {
-  int32_t functionId;                    // system assigned function id
-  int32_t funcType;                      // scalar function or aggregate function
-  int8_t  resType;                       // result type
-  int16_t resBytes;                      // result byte
-  int32_t contLen;                       // content length
-  int32_t bufSize;                       // interbuf size
-  char*   name;                          // function name
-  void*   handle;                        // handle loaded in mem
-  void*   funcs[TSDB_UDF_FUNC_MAX_NUM];  // function ptr
-
-  // for script like lua/javascript only
-  int   isScript;
-  void* pScriptCtx;
-
-  SUdfInit init;
-  char*    content;
-  char*    path;
+  char   *udfName;        // function name
+  int32_t udfType;    // scalar function or aggregate function
+  int8_t    scriptType;
+  char *path;
+
+  int8_t  resType;     // result type
+  int16_t resBytes;    // result byte
+  int32_t bufSize;     //interbuf size
+
 } SUdfInfo;
 
+typedef void *UdfHandle;
+
+/**
+ * setup udf
+ * @param udf, in
+ * @param handle, out
+ * @return error code
+ */
+int32_t setupUdf(SUdfInfo* udf, UdfHandle *handle);
+
+
+enum {
+  TSDB_UDF_STEP_NORMAL = 0,
+  TSDB_UDF_STEP_MERGE,
+  TSDb_UDF_STEP_FINALIZE,
+  TSDB_UDF_STEP_MAX_NUM
+};
+/**
+ * call udf
+ * @param handle udf handle
+ * @param step
+ * @param state
+ * @param stateSize
+ * @param input
+ * @param newstate
+ * @param newStateSize
+ * @param output
+ * @return error code
+ */
+
+//TODO: must change the following after metadata flow and data flow between qworker and udfd is well defined
+typedef struct SUdfDataBlock {
+  char* data;
+  int32_t size;
+} SUdfDataBlock;
+
+int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SUdfDataBlock input, char **newstate,
+                int32_t *newStateSize, SUdfDataBlock *output);
+
+/**
+ * tearn down udf
+ * @param handle
+ * @return
+ */
+int32_t teardownUdf(UdfHandle handle);
+
+// end API to taosd and qworker
+//=============================================================================================================================
+// TODO: Must change
+// begin API to UDF writer.
+
 // script
 
-typedef int32_t (*scriptInitFunc)(void* pCtx);
-typedef void (*scriptNormalFunc)(void* pCtx, char* data, int16_t iType, int16_t iBytes, int32_t numOfRows,
-                                 int64_t* ptList, int64_t key, char* dataOutput, char* tsOutput, int32_t* numOfOutput,
-                                 int16_t oType, int16_t oBytes);
-typedef void (*scriptFinalizeFunc)(void* pCtx, int64_t key, char* dataOutput, int32_t* numOfOutput);
-typedef void (*scriptMergeFunc)(void* pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput);
-typedef void (*scriptDestroyFunc)(void* pCtx);
+//typedef int32_t (*scriptInitFunc)(void* pCtx);
+//typedef void (*scriptNormalFunc)(void* pCtx, char* data, int16_t iType, int16_t iBytes, int32_t numOfRows,
+//                                 int64_t* ptList, int64_t key, char* dataOutput, char* tsOutput, int32_t* numOfOutput,
+//                                 int16_t oType, int16_t oBytes);
+//typedef void (*scriptFinalizeFunc)(void* pCtx, int64_t key, char* dataOutput, int32_t* numOfOutput);
+//typedef void (*scriptMergeFunc)(void* pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput);
+//typedef void (*scriptDestroyFunc)(void* pCtx);
 
 // dynamic lib
-typedef void (*udfNormalFunc)(char* data, int16_t itype, int16_t iBytes, int32_t numOfRows, int64_t* ts,
-                              char* dataOutput, char* interBuf, char* tsOutput, int32_t* numOfOutput, int16_t oType,
-                              int16_t oBytes, SUdfInit* buf);
-typedef int32_t (*udfInitFunc)(SUdfInit* data);
-typedef void (*udfFinalizeFunc)(char* dataOutput, char* interBuf, int32_t* numOfOutput, SUdfInit* buf);
-typedef void (*udfMergeFunc)(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf);
-typedef void (*udfDestroyFunc)(SUdfInit* buf);
+typedef int32_t (*TUdfInitFunc)();
+typedef void (*TUdfDestroyFunc)();
+
+typedef void (*TUdfFunc)(int8_t step,
+                         char *state, int32_t stateSize, SUdfDataBlock input,
+                         char **newstate, int32_t *newStateSize, SUdfDataBlock *output);
+
+//typedef void (*udfMergeFunc)(char *data, int32_t numOfRows, char *dataOutput, int32_t* numOfOutput);
+//typedef void (*udfFinalizeFunc)(char* state, int32_t stateSize, SUdfDataBlock *output);
+
+// end API to UDF writer
+//=======================================================================================================================
 
 #ifdef __cplusplus
 }

source/libs/function/inc/tudfInt.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TUDF_INT_H
+#define TDENGINE_TUDF_INT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//TODO replaces them with fnDebug
+//#define debugPrint(...) taosPrintLog("Function", DEBUG_INFO,  135, __VA_ARGS__)
+#define debugPrint(...) {fprintf(stderr, __VA_ARGS__);fprintf(stderr, "\n");}
+enum {
+  UDF_TASK_SETUP = 0,
+  UDF_TASK_CALL = 1,
+  UDF_TASK_TEARDOWN = 2
+
+};
+
+typedef struct SUdfSetupRequest {
+  char udfName[16]; //
+  int8_t scriptType; // 0:c, 1: lua, 2:js
+  int8_t udfType; //udaf, udf
+  int16_t pathSize;
+  char *path;
+} SUdfSetupRequest;
+
+typedef struct SUdfSetupResponse {
+  int64_t udfHandle;
+} SUdfSetupResponse;
+
+
+typedef struct SUdfCallRequest {
+  int64_t udfHandle;
+  int8_t step;
+
+  int32_t inputBytes;
+  char *input;
+
+  int32_t stateBytes;
+  char *state;
+} SUdfCallRequest;
+
+
+typedef struct SUdfCallResponse {
+  int32_t outputBytes;
+  char *output;
+  int32_t newStateBytes;
+  char *newState;
+} SUdfCallResponse;
+
+
+typedef struct SUdfTeardownRequest {
+  int64_t udfHandle;
+} SUdfTeardownRequest;
+
+
+typedef struct SUdfTeardownResponse {
+} SUdfTeardownResponse;
+
+typedef struct SUdfRequest {
+  int32_t msgLen;
+  int64_t seqNum;
+
+  int8_t type;
+  void *subReq;
+} SUdfRequest;
+
+typedef struct SUdfResponse {
+  int32_t msgLen;
+  int64_t seqNum;
+
+  int8_t type;
+  int32_t code;
+  void *subRsp;
+} SUdfResponse;
+
+int32_t decodeRequest(char *buf, int32_t bufLen, SUdfRequest **pRequest);
+int32_t encodeResponse(char **buf, int32_t *bufLen, SUdfResponse *response);
+int32_t encodeRequest(char **buf, int32_t *bufLen, SUdfRequest *request);
+int32_t decodeResponse(char *buf, int32_t bufLen, SUdfResponse **pResponse);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TUDF_INT_H

source/libs/function/inc/udfc.h

+//
+// Created by shenglian on 28/02/22.
+//
+
+#ifndef UDF_UDF_H
+#define UDF_UDF_H
+
+#include <stdlib.h>
+#define DEBUG
+#ifdef DEBUG
+#define debugPrint(...) fprintf(__VA_ARGS__)
+#else
+#define debugPrint(...) /**/
+#endif
+
+enum {
+    UDF_TASK_SETUP = 0,
+    UDF_TASK_CALL = 1,
+    UDF_TASK_TEARDOWN = 2
+
+};
+
+typedef struct SSDataBlock{
+    char *data;
+    int32_t size;
+} SSDataBlock;
+
+typedef struct SUdfInfo {
+    char *udfName;
+    char *path;
+} SUdfInfo;
+
+typedef void *UdfHandle;
+
+int32_t startUdfService();
+
+int32_t stopUdfService();
+
+//int32_t setupUdf(SUdfInfo *udf, int32_t numOfUdfs, UdfHandle *handles);
+
+int32_t setupUdf(SUdfInfo* udf, UdfHandle* handle);
+
+int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SSDataBlock input, char **newstate,
+                int32_t *newStateSize, SSDataBlock *output);
+
+int32_t teardownUdf(UdfHandle handle);
+
+typedef struct SUdfSetupRequest {
+    char udfName[16]; //
+    int8_t scriptType; // 0:c, 1: lua, 2:js
+    int8_t udfType; //udaf, udf, udtf
+    int16_t pathSize;
+    char *path;
+} SUdfSetupRequest;
+
+typedef struct SUdfSetupResponse {
+    int64_t udfHandle;
+} SUdfSetupResponse;
+
+
+typedef struct SUdfCallRequest {
+    int64_t udfHandle;
+    int8_t step;
+
+    int32_t inputBytes;
+    char *input;
+
+    int32_t stateBytes;
+    char *state;
+} SUdfCallRequest;
+
+
+typedef struct SUdfCallResponse {
+    int32_t outputBytes;
+    char *output;
+    int32_t newStateBytes;
+    char *newState;
+} SUdfCallResponse;
+
+
+typedef struct SUdfTeardownRequest {
+    int64_t udfHandle;
+} SUdfTeardownRequest;
+
+
+typedef struct SUdfTeardownResponse {
+} SUdfTeardownResponse;
+
+typedef struct SUdfRequest {
+    int32_t msgLen;
+    int64_t seqNum;
+
+    int8_t type;
+    void *subReq;
+} SUdfRequest;
+
+typedef struct SUdfResponse {
+    int32_t msgLen;
+    int64_t seqNum;
+
+    int8_t type;
+    int32_t code;
+    void *subRsp;
+} SUdfResponse;
+
+int32_t decodeRequest(char *buf, int32_t bufLen, SUdfRequest **pRequest);
+int32_t encodeResponse(char **buf, int32_t *bufLen, SUdfResponse *response);
+int32_t encodeRequest(char **buf, int32_t *bufLen, SUdfRequest *request);
+int32_t decodeResponse(char *buf, int32_t bufLen, SUdfResponse **pResponse);
+#endif //UDF_UDF_H

source/libs/function/src/udfd.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "uv.h"
+#include "os.h"
+#include "tlog.h"
+
+#include "tudf.h"
+#include "tudfInt.h"
+
+
+static uv_loop_t *loop;
+
+typedef struct SUdfdUvConn {
+    uv_stream_t *client;
+    char *inputBuf;
+    int32_t inputLen;
+    int32_t inputCap;
+    int32_t inputTotal;
+} SUdfdUvConn;
+
+typedef struct SUvUdfWork {
+    uv_stream_t *client;
+    uv_buf_t input;
+    uv_buf_t output;
+} SUvUdfWork;
+
+typedef struct SUdf {
+    int32_t refCount;
+
+    char name[16];
+    int8_t type;
+
+    uv_lib_t lib;
+    TUdfFunc normalFunc;
+} SUdf;
+
+//TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix
+//TODO: add private udf structure.
+typedef struct SUdfHandle {
+    SUdf *udf;
+} SUdfHandle;
+
+
+void udfdProcessRequest(uv_work_t *req) {
+    SUvUdfWork *uvUdf = (SUvUdfWork *) (req->data);
+    SUdfRequest *request = NULL;
+    decodeRequest(uvUdf->input.base, uvUdf->input.len, &request);
+
+    switch (request->type) {
+        case UDF_TASK_SETUP: {
+            debugPrint("%s", "process setup request");
+            SUdf *udf = malloc(sizeof(SUdf));
+            udf->refCount = 0;
+            SUdfSetupRequest *setup = request->subReq;
+            strcpy(udf->name, setup->udfName);
+            int err = uv_dlopen(setup->path, &udf->lib);
+            if (err != 0) {
+                debugPrint("can not load library %s. error: %s", setup->path, uv_strerror(err));
+                //TODO set error
+            }
+
+            char normalFuncName[32] = {0};
+            strcpy(normalFuncName, setup->udfName);
+	    //TODO error,
+	    //TODO find all functions normal, init, destroy, normal, merge, finalize
+            uv_dlsym(&udf->lib, normalFuncName, (void **) (&udf->normalFunc));
+
+            SUdfHandle *handle = malloc(sizeof(SUdfHandle));
+            handle->udf = udf;
+            udf->refCount++;
+            //TODO: allocate private structure and call init function and set it to handle
+            SUdfResponse *rsp = malloc(sizeof(SUdfResponse));
+            rsp->seqNum = request->seqNum;
+            rsp->type = request->type;
+            rsp->code = 0;
+            SUdfSetupResponse *subRsp = malloc(sizeof(SUdfSetupResponse));
+            subRsp->udfHandle = (int64_t) (handle);
+            rsp->subRsp = subRsp;
+            char *buf;
+            int32_t len;
+            encodeResponse(&buf, &len, rsp);
+
+            uvUdf->output = uv_buf_init(buf, len);
+
+            free(rsp->subRsp);
+            free(rsp);
+            free(request->subReq);
+            free(request);
+            free(uvUdf->input.base);
+            break;
+        }
+
+        case UDF_TASK_CALL: {
+            debugPrint("%s", "process call request");
+            SUdfCallRequest *call = request->subReq;
+            SUdfHandle *handle = (SUdfHandle *) (call->udfHandle);
+            SUdf *udf = handle->udf;
+            char *newState;
+            int32_t newStateSize;
+            SUdfDataBlock input = {.data = call->input, .size= call->inputBytes};
+            SUdfDataBlock output;
+	    //TODO: call different functions according to the step 
+            udf->normalFunc(call->step, call->state, call->stateBytes, input, &newState, &newStateSize, &output);
+
+            SUdfResponse *rsp = malloc(sizeof(SUdfResponse));
+            rsp->seqNum = request->seqNum;
+            rsp->type = request->type;
+            rsp->code = 0;
+            SUdfCallResponse *subRsp = malloc(sizeof(SUdfCallResponse));
+            subRsp->outputBytes = output.size;
+            subRsp->output = output.data;
+            subRsp->newStateBytes = newStateSize;
+            subRsp->newState = newState;
+            rsp->subRsp = subRsp;
+
+            char *buf;
+            int32_t len;
+            encodeResponse(&buf, &len, rsp);
+            uvUdf->output = uv_buf_init(buf, len);
+
+            free(rsp->subRsp);
+            free(rsp);
+            free(newState);
+            free(output.data);
+            free(request->subReq);
+            free(request);
+            free(uvUdf->input.base);
+            break;
+        }
+        case UDF_TASK_TEARDOWN: {
+            debugPrint("%s", "process teardown request");
+
+            SUdfTeardownRequest *teardown = request->subReq;
+            SUdfHandle *handle = (SUdfHandle *) (teardown->udfHandle);
+            SUdf *udf = handle->udf;
+            udf->refCount--;
+            if (udf->refCount == 0) {
+                uv_dlclose(&udf->lib);
+            }
+            free(udf);
+	    //TODO: call destroy and free udf private 
+            free(handle);
+
+            SUdfResponse *rsp = malloc(sizeof(SUdfResponse));
+            rsp->seqNum = request->seqNum;
+            rsp->type = request->type;
+            rsp->code = 0;
+            SUdfTeardownResponse *subRsp = malloc(sizeof(SUdfTeardownResponse));
+            rsp->subRsp = subRsp;
+            char *buf;
+            int32_t len;
+            encodeResponse(&buf, &len, rsp);
+            uvUdf->output = uv_buf_init(buf, len);
+
+            free(rsp->subRsp);
+            free(rsp);
+            free(request->subReq);
+            free(request);
+            free(uvUdf->input.base);
+            break;
+        }
+        default: {
+            break;
+        }
+
+    }
+
+}
+
+void udfdOnWrite(uv_write_t *req, int status) {
+    debugPrint("%s", "after writing to pipe");
+    if (status < 0) {
+        debugPrint("Write error %s", uv_err_name(status));
+    }
+    SUvUdfWork *work = (SUvUdfWork *) req->data;
+    debugPrint("\tlength: %zu", work->output.len);
+    free(work->output.base);
+    free(work);
+    free(req);
+}
+
+
+void udfdSendResponse(uv_work_t *work, int status) {
+    debugPrint("%s", "send response");
+    SUvUdfWork *udfWork = (SUvUdfWork *) (work->data);
+
+    uv_write_t *write_req = malloc(sizeof(uv_write_t));
+    write_req->data = udfWork;
+    uv_write(write_req, udfWork->client, &udfWork->output, 1, udfdOnWrite);
+
+    free(work);
+}
+
+void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) {
+    debugPrint("%s", "allocate buffer for read");
+    SUdfdUvConn *ctx = handle->data;
+    int32_t msgHeadSize = sizeof(int32_t) + sizeof(int64_t);
+    if (ctx->inputCap == 0) {
+        ctx->inputBuf = malloc(msgHeadSize);
+        if (ctx->inputBuf) {
+            ctx->inputLen = 0;
+            ctx->inputCap = msgHeadSize;
+            ctx->inputTotal = -1;
+
+            buf->base = ctx->inputBuf;
+            buf->len = ctx->inputCap;
+        } else {
+            //TODO: log error
+            buf->base = NULL;
+            buf->len = 0;
+        }
+    } else {
+        ctx->inputCap = ctx->inputTotal > ctx->inputCap ? ctx->inputTotal : ctx->inputCap;
+        void *inputBuf = realloc(ctx->inputBuf, ctx->inputCap);
+        if (inputBuf) {
+            ctx->inputBuf = inputBuf;
+            buf->base = ctx->inputBuf + ctx->inputLen;
+            buf->len = ctx->inputCap - ctx->inputLen;
+        } else {
+            //TODO: log error
+            buf->base = NULL;
+            buf->len = 0;
+        }
+    }
+    debugPrint("\tinput buf cap - len - total : %d - %d - %d", ctx->inputCap, ctx->inputLen, ctx->inputTotal);
+
+}
+
+bool isUdfdUvMsgComplete(SUdfdUvConn *pipe) {
+    if (pipe->inputTotal == -1 && pipe->inputLen >= sizeof(int32_t)) {
+        pipe->inputTotal = *(int32_t *) (pipe->inputBuf);
+    }
+    if (pipe->inputLen == pipe->inputCap && pipe->inputTotal == pipe->inputCap) {
+        return true;
+    }
+    return false;
+}
+
+void udfdHandleRequest(SUdfdUvConn *conn) {
+    uv_work_t *work = malloc(sizeof(uv_work_t));
+    SUvUdfWork *udfWork = malloc(sizeof(SUvUdfWork));
+    udfWork->client = conn->client;
+    udfWork->input = uv_buf_init(conn->inputBuf, conn->inputLen);
+    conn->inputBuf = NULL;
+    conn->inputLen = 0;
+    conn->inputCap = 0;
+    conn->inputTotal = -1;
+    work->data = udfWork;
+    uv_queue_work(loop, work, udfdProcessRequest, udfdSendResponse);
+}
+
+void udfdPipeCloseCb(uv_handle_t *pipe) {
+    SUdfdUvConn *conn = pipe->data;
+    free(conn->client);
+    free(conn->inputBuf);
+    free(conn);
+}
+
+void udfdUvHandleError(SUdfdUvConn *conn) {
+    uv_close((uv_handle_t *) conn->client, udfdPipeCloseCb);
+}
+
+void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
+    debugPrint("%s, nread: %zd", "read from pipe", nread);
+
+    if (nread == 0) return;
+
+    SUdfdUvConn *conn = client->data;
+
+    if (nread > 0) {
+        conn->inputLen += nread;
+        if (isUdfdUvMsgComplete(conn)) {
+            udfdHandleRequest(conn);
+        } else {
+            //log error or continue;
+        }
+        return;
+    }
+
+    if (nread < 0) {
+        debugPrint("Read error %s", uv_err_name(nread));
+        if (nread == UV_EOF) {
+            //TODO check more when close
+        } else {
+        }
+        udfdUvHandleError(conn);
+    }
+}
+
+void udfdOnNewConnection(uv_stream_t *server, int status) {
+    debugPrint("%s", "on new connection");
+    if (status < 0) {
+        // TODO
+        return;
+    }
+
+    uv_pipe_t *client = (uv_pipe_t *) malloc(sizeof(uv_pipe_t));
+    uv_pipe_init(loop, client, 0);
+    if (uv_accept(server, (uv_stream_t *) client) == 0) {
+        SUdfdUvConn *ctx = malloc(sizeof(SUdfdUvConn));
+        ctx->client = (uv_stream_t *) client;
+        ctx->inputBuf = 0;
+        ctx->inputLen = 0;
+        ctx->inputCap = 0;
+        client->data = ctx;
+        ctx->client = (uv_stream_t *) client;
+        uv_read_start((uv_stream_t *) client, udfdAllocBuffer, udfdPipeRead);
+    } else {
+        uv_close((uv_handle_t *) client, NULL);
+    }
+}
+
+void removeListeningPipe(int sig) {
+    uv_fs_t req;
+    uv_fs_unlink(loop, &req, "udf.sock", NULL);
+    exit(0);
+}
+
+int main() {
+    debugPrint("libuv version: %x", UV_VERSION_HEX);
+
+    loop = uv_default_loop();
+    uv_fs_t req;
+    uv_fs_unlink(loop, &req, "udf.sock", NULL);
+
+    uv_pipe_t server;
+    uv_pipe_init(loop, &server, 0);
+
+    signal(SIGINT, removeListeningPipe);
+
+    int r;
+    if ((r = uv_pipe_bind(&server, "udf.sock"))) {
+        debugPrint("Bind error %s\n", uv_err_name(r));
+        removeListeningPipe(0);
+        return 1;
+    }
+    if ((r = uv_listen((uv_stream_t *) &server, 128, udfdOnNewConnection))) {
+        debugPrint("Listen error %s", uv_err_name(r));
+        return 2;
+    }
+    uv_run(loop, UV_RUN_DEFAULT);
+    uv_loop_close(loop);
+}

source/libs/function/test/runUdf.c

+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "uv.h"
+#include "tudf.h"
+
+int main(int argc, char *argv[]) {
+    startUdfService();
+    uv_sleep(1000);
+    char path[256] = {0};
+    size_t cwdSize = 256;
+    int err = uv_cwd(path, &cwdSize);
+    if (err != 0) {
+        fprintf(stderr, "err cwd: %s\n", uv_strerror(err));
+	    return err;
+    }
+    fprintf(stdout, "current working directory:%s\n", path);
+    strcat(path, "/libudf1.so");
+    SUdfInfo udfInfo = {.udfName="udf1", .path=path};
+
+    UdfHandle handle;
+    setupUdf(&udfInfo, &handle);
+
+    //char state[5000000] = "state";
+    //char input[5000000] = "input";
+    int dataSize = 500;
+    int callCount = 2;
+    if (argc > 1) dataSize = atoi(argv[1]);
+    if (argc > 2) callCount = atoi(argv[2]);
+    char *state = malloc(dataSize);
+    char *input = malloc(dataSize);
+    SUdfDataBlock blockInput = {.data = input, .size = dataSize};
+    SUdfDataBlock blockOutput;
+    char* newState;
+    int32_t newStateSize;
+    for (int l = 0; l < callCount; ++l) {
+        callUdf(handle, 0, state, dataSize, blockInput, &newState, &newStateSize, &blockOutput);
+    }
+    free(state);
+    free(input);
+    teardownUdf(handle);
+
+    stopUdfService();
+}

source/libs/function/test/udf1.c

+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "tudf.h"
+
+void udf1(int8_t step, char *state, int32_t stateSize, SUdfDataBlock input,
+          char **newState, int32_t *newStateSize, SUdfDataBlock *output) {
+    fprintf(stdout, "%s, step:%d\n", "udf function called", step);
+    char *newStateBuf = malloc(stateSize);
+    memcpy(newStateBuf, state, stateSize);
+    *newState = newStateBuf;
+    *newStateSize = stateSize;
+
+    char *outputBuf = malloc(input.size);
+    memcpy(outputBuf, input.data, input.size);
+    output->data = outputBuf;
+    output->size = input.size;
+    return;
+}

source/libs/index/src/index_fst_automation.c

@@ -154,15 +154,7 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) {
     // add more search type
   }
 
-  char* dst = NULL;
-  if (data != NULL) {
-    char*  src = (char*)data;
-    size_t len = strlen(src);
-    dst = (char*)calloc(1, len * sizeof(char) + 1);
-    memcpy(dst, src, len);
-  }
-
-  ctx->data = dst;
+  ctx->data = strdup((char*)data);
   ctx->type = atype;
   ctx->stdata = (void*)sv;
   return ctx;

source/libs/nodes/src/nodesCodeFuncs.c

@@ -1480,7 +1480,7 @@ static int32_t jsonToDatum(const SJson* pJson, void* pObj) {
     case TSDB_DATA_TYPE_NCHAR:
     case TSDB_DATA_TYPE_VARCHAR:
     case TSDB_DATA_TYPE_VARBINARY: {
-      pNode->datum.p = calloc(1, pNode->node.resType.bytes + VARSTR_HEADER_SIZE + 1 + 100);
+      pNode->datum.p = calloc(1, pNode->node.resType.bytes + VARSTR_HEADER_SIZE + 1);
       if (NULL == pNode->datum.p) {
         code = TSDB_CODE_OUT_OF_MEMORY;
         break;

source/libs/parser/inc/parAst.h

@@ -150,9 +150,9 @@ SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName);
 SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort);
 SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode);
 SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, const SToken* pValue);
-SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, SToken* pIndexName, SToken* pTableName, SNodeList* pCols, SNode* pOptions);
+SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SToken* pIndexName, SToken* pTableName, SNodeList* pCols, SNode* pOptions);
 SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInterval, SNode* pOffset, SNode* pSliding);
-SNode* createDropIndexStmt(SAstCreateContext* pCxt, SToken* pIndexName, SToken* pTableName);
+SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pIndexName, SToken* pTableName);
 SNode* createCreateQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId);
 SNode* createDropQnodeStmt(SAstCreateContext* pCxt, const SToken* pDnodeId);
 SNode* createCreateTopicStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pQuery, const SToken* pSubscribeDbName);

source/libs/parser/inc/sql.y

@@ -313,10 +313,11 @@ func_name_list(A) ::= func_name_list(B) NK_COMMA col_name(C).
 func_name(A) ::= function_name(B).                                                { A = createFunctionNode(pCxt, &B, NULL); }
 
 /************************************************ create index ********************************************************/
-cmd ::= CREATE SMA INDEX index_name(A) ON table_name(B) index_options(C).         { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, &A, &B, NULL, C); }
-cmd ::= CREATE FULLTEXT INDEX
-  index_name(A) ON table_name(B) NK_LP col_name_list(C) NK_RP.                    { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, &A, &B, C, NULL); }
-cmd ::= DROP INDEX index_name(A) ON table_name(B).                                { pCxt->pRootNode = createDropIndexStmt(pCxt, &A, &B); }
+cmd ::= CREATE SMA INDEX not_exists_opt(D) 
+  index_name(A) ON table_name(B) index_options(C).                                { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, D, &A, &B, NULL, C); }
+cmd ::= CREATE FULLTEXT INDEX not_exists_opt(D)
+  index_name(A) ON table_name(B) NK_LP col_name_list(C) NK_RP.                    { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, D, &A, &B, C, NULL); }
+cmd ::= DROP INDEX exists_opt(C) index_name(A) ON table_name(B).                  { pCxt->pRootNode = createDropIndexStmt(pCxt, C, &A, &B); }
 
 index_options(A) ::= .                                                            { A = NULL; }
 index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL 

source/libs/parser/src/parAstCreater.c

@@ -1160,13 +1160,14 @@ SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const
   return (SNode*)pStmt;
 }
 
-SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, SToken* pIndexName, SToken* pTableName, SNodeList* pCols, SNode* pOptions) {
+SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SToken* pIndexName, SToken* pTableName, SNodeList* pCols, SNode* pOptions) {
   if (!checkIndexName(pCxt, pIndexName) || !checkTableName(pCxt, pTableName)) {
     return NULL;
   }
   SCreateIndexStmt* pStmt = nodesMakeNode(QUERY_NODE_CREATE_INDEX_STMT);
   CHECK_OUT_OF_MEM(pStmt);
   pStmt->indexType = type;
+  pStmt->ignoreExists = ignoreExists;
   strncpy(pStmt->indexName, pIndexName->z, pIndexName->n);
   strncpy(pStmt->tableName, pTableName->z, pTableName->n);
   pStmt->pCols = pCols;
@@ -1184,12 +1185,13 @@ SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInt
   return (SNode*)pOptions;
 }
 
-SNode* createDropIndexStmt(SAstCreateContext* pCxt, SToken* pIndexName, SToken* pTableName) {
+SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pIndexName, SToken* pTableName) {
   if (!checkIndexName(pCxt, pIndexName) || !checkTableName(pCxt, pTableName)) {
     return NULL;
   }
   SDropIndexStmt* pStmt = nodesMakeNode(QUERY_NODE_DROP_INDEX_STMT);
   CHECK_OUT_OF_MEM(pStmt);
+  pStmt->ignoreNotExists = ignoreNotExists;
   strncpy(pStmt->indexName, pIndexName->z, pIndexName->n);
   strncpy(pStmt->tableName, pTableName->z, pTableName->n);
   return (SNode*)pStmt;

source/libs/parser/src/parTranslater.c

@@ -84,12 +84,18 @@ static SName* toName(int32_t acctId, const char* pDbName, const char* pTableName
   return pName;
 }
 
-static int32_t collectUseDatabase(const char* pFullDbName, SHashObj* pDbs) {
+static int32_t collectUseDatabaseImpl(const char* pFullDbName, SHashObj* pDbs) {
   SFullDatabaseName name = {0};
   strcpy(name.fullDbName, pFullDbName);
   return taosHashPut(pDbs, pFullDbName, strlen(pFullDbName), &name, sizeof(SFullDatabaseName));
 }
 
+static int32_t collectUseDatabase(const SName* pName, SHashObj* pDbs) {
+  char dbFName[TSDB_DB_FNAME_LEN] = {0};
+  tNameGetFullDbName(pName, dbFName);
+  return collectUseDatabaseImpl(dbFName, pDbs);
+}
+
 static int32_t collectUseTable(const SName* pName, SHashObj* pDbs) {
   char fullName[TSDB_TABLE_FNAME_LEN];
   tNameExtractFullName(pName, fullName);
@@ -98,7 +104,10 @@ static int32_t collectUseTable(const SName* pName, SHashObj* pDbs) {
 
 static int32_t getTableMetaImpl(STranslateContext* pCxt, const SName* pName, STableMeta** pMeta) {
   SParseContext* pParCxt = pCxt->pParseCxt;
-  int32_t code = collectUseTable(pName, pCxt->pTables);
+  int32_t code = collectUseDatabase(pName, pCxt->pDbs);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = collectUseTable(pName, pCxt->pTables);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = catalogGetTableMeta(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, pName, pMeta);
   }
@@ -117,7 +126,10 @@ static int32_t getTableMeta(STranslateContext* pCxt, const char* pDbName, const
 
 static int32_t getTableDistVgInfo(STranslateContext* pCxt, const SName* pName, SArray** pVgInfo) {
   SParseContext* pParCxt = pCxt->pParseCxt;
-  int32_t code = collectUseTable(pName, pCxt->pTables);
+  int32_t code = collectUseDatabase(pName, pCxt->pDbs);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = collectUseTable(pName, pCxt->pTables);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = catalogGetTableDistVgInfo(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, pName, pVgInfo);
   }
@@ -131,7 +143,7 @@ static int32_t getDBVgInfoImpl(STranslateContext* pCxt, const SName* pName, SArr
   SParseContext* pParCxt = pCxt->pParseCxt;
   char fullDbName[TSDB_DB_FNAME_LEN];
   tNameGetFullDbName(pName, fullDbName);
-  int32_t code = collectUseDatabase(fullDbName, pCxt->pDbs);
+  int32_t code = collectUseDatabaseImpl(fullDbName, pCxt->pDbs);
   if (TSDB_CODE_SUCCESS == code) {
     code = catalogGetDBVgInfo(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, fullDbName, pVgInfo);
   }
@@ -151,7 +163,10 @@ static int32_t getDBVgInfo(STranslateContext* pCxt, const char* pDbName, SArray*
 
 static int32_t getTableHashVgroupImpl(STranslateContext* pCxt, const SName* pName, SVgroupInfo* pInfo) {
   SParseContext* pParCxt = pCxt->pParseCxt;
-  int32_t code = collectUseTable(pName, pCxt->pTables);
+  int32_t code = collectUseDatabase(pName, pCxt->pDbs);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = collectUseTable(pName, pCxt->pTables);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = catalogGetTableHashVgroup(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, pName, pInfo);
   }
@@ -170,7 +185,7 @@ static int32_t getTableHashVgroup(STranslateContext* pCxt, const char* pDbName,
 
 static int32_t getDBVgVersion(STranslateContext* pCxt, const char* pDbFName, int32_t* pVersion, int64_t* pDbId, int32_t* pTableNum) {
   SParseContext* pParCxt = pCxt->pParseCxt;
-  int32_t code = collectUseDatabase(pDbFName, pCxt->pDbs);
+  int32_t code = collectUseDatabaseImpl(pDbFName, pCxt->pDbs);
   if (TSDB_CODE_SUCCESS == code) {
     code = catalogGetDBVgVersion(pParCxt->pCatalog, pDbFName, pVersion, pDbId, pTableNum);
   }
@@ -1407,34 +1422,110 @@ static int32_t translateShowTables(STranslateContext* pCxt) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t translateCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) {
-  SVCreateTSmaReq createSmaReq = {0};
+static int32_t getSmaIndexDstVgId(STranslateContext* pCxt, char* pTableName, int32_t* pVgId) {
+  SVgroupInfo vg = {0};
+  int32_t code = getTableHashVgroup(pCxt, pCxt->pParseCxt->db, pTableName, &vg);
+  if (TSDB_CODE_SUCCESS == code) {
+    *pVgId = vg.vgId;
+  }
+  return code;
+}
 
-  if (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pInterval) ||
-      (NULL != pStmt->pOptions->pOffset && DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pOffset)) ||
-      (NULL != pStmt->pOptions->pSliding && DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pSliding))) {
-    return pCxt->errCode;
+static int32_t getSmaIndexSql(STranslateContext* pCxt, char** pSql, int32_t* pLen) {
+  *pSql = strdup(pCxt->pParseCxt->pSql);
+  if (NULL == *pSql) {
+    return TSDB_CODE_OUT_OF_MEMORY;
   }
+  *pLen = pCxt->pParseCxt->sqlLen + 1;
+  return TSDB_CODE_SUCCESS;
+}
 
-  createSmaReq.tSma.intervalUnit = ((SValueNode*)pStmt->pOptions->pInterval)->unit;
-  createSmaReq.tSma.slidingUnit = (NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->unit : 0);
-  strcpy(createSmaReq.tSma.indexName, pStmt->indexName);
+static int32_t getSmaIndexExpr(STranslateContext* pCxt, SCreateIndexStmt* pStmt, char** pExpr, int32_t* pLen) {
+  return nodesListToString(pStmt->pOptions->pFuncs, false, pExpr, pLen);
+}
 
-  SName name;
-  name.type = TSDB_TABLE_NAME_T;
-  name.acctId = pCxt->pParseCxt->acctId;
+static int32_t getSmaIndexBuildAst(STranslateContext* pCxt, SCreateIndexStmt* pStmt, char** pAst, int32_t* pLen) {
+  SSelectStmt* pSelect = nodesMakeNode(QUERY_NODE_SELECT_STMT);
+  if (NULL == pSelect) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  SRealTableNode* pTable = nodesMakeNode(QUERY_NODE_REAL_TABLE);
+  if (NULL == pTable) {
+    nodesDestroyNode(pSelect);
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  strcpy(pTable->table.dbName, pCxt->pParseCxt->db);
+  strcpy(pTable->table.tableName, pStmt->tableName);
+  pSelect->pFromTable = (SNode*)pTable;
+
+  pSelect->pProjectionList = nodesCloneList(pStmt->pOptions->pFuncs);
+  if (NULL == pTable) {
+    nodesDestroyNode(pSelect);
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  SIntervalWindowNode* pInterval = nodesMakeNode(QUERY_NODE_INTERVAL_WINDOW);
+  if (NULL == pInterval) {
+    nodesDestroyNode(pSelect);
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  pSelect->pWindow = (SNode*)pInterval;
+  pInterval->pInterval = nodesCloneNode(pStmt->pOptions->pInterval);
+  pInterval->pOffset = nodesCloneNode(pStmt->pOptions->pOffset);
+  pInterval->pSliding = nodesCloneNode(pStmt->pOptions->pSliding);
+  if (NULL == pInterval->pInterval || (NULL != pStmt->pOptions->pOffset && NULL == pInterval->pOffset) ||
+      (NULL != pStmt->pOptions->pSliding && NULL == pInterval->pSliding)) {
+    nodesDestroyNode(pSelect);
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  int32_t code = translateQuery(pCxt, (SNode*)pSelect);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = nodesNodeToString(pSelect, false, pAst, pLen);
+  }
+  nodesDestroyNode(pSelect);
+  return code;
+}
+
+static int32_t buildCreateSmaReq(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SMCreateSmaReq* pReq) {
+  SName name = { .type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId };
   strcpy(name.dbname, pCxt->pParseCxt->db);
+  strcpy(name.tname, pStmt->indexName);
+  tNameExtractFullName(&name, pReq->name);
   strcpy(name.tname, pStmt->tableName);
-  STableMeta* pMeta = NULL;
-  int32_t code = catalogGetTableMeta(pCxt->pParseCxt->pCatalog, pCxt->pParseCxt->pTransporter, &pCxt->pParseCxt->mgmtEpSet, &name, &pMeta);
-  if (TSDB_CODE_SUCCESS != code) {
-    return code;
+  name.tname[strlen(pStmt->tableName)] = '\0';
+  tNameExtractFullName(&name, pReq->stb);
+  pReq->igExists = pStmt->ignoreExists;
+  pReq->interval = ((SValueNode*)pStmt->pOptions->pInterval)->datum.i;
+  pReq->intervalUnit = ((SValueNode*)pStmt->pOptions->pInterval)->unit;
+  pReq->offset = (NULL != pStmt->pOptions->pOffset ? ((SValueNode*)pStmt->pOptions->pOffset)->datum.i : 0);
+  pReq->sliding = (NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->datum.i : pReq->interval);
+  pReq->slidingUnit = (NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->unit : pReq->intervalUnit);
+
+  int32_t code = getSmaIndexDstVgId(pCxt, pStmt->tableName, &pReq->dstVgId);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = getSmaIndexSql(pCxt, &pReq->sql, &pReq->sqlLen);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = getSmaIndexExpr(pCxt, pStmt, &pReq->expr, &pReq->exprLen);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = getSmaIndexBuildAst(pCxt, pStmt, &pReq->ast, &pReq->astLen);
+  }
+
+  return code;
+}
 
-  createSmaReq.tSma.tableUid = pMeta->uid;
-  createSmaReq.tSma.interval = ((SValueNode*)pStmt->pOptions->pInterval)->datum.i;
-  createSmaReq.tSma.sliding = (NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->datum.i : 0);
-  code = nodesListToString(pStmt->pOptions->pFuncs, false, &createSmaReq.tSma.expr, &createSmaReq.tSma.exprLen);
+static int32_t translateCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) {
+  if (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pInterval) ||
+      (NULL != pStmt->pOptions->pOffset && DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pOffset)) ||
+      (NULL != pStmt->pOptions->pSliding && DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pStmt->pOptions->pSliding))) {
+    return pCxt->errCode;
+  }
+
+  SMCreateSmaReq createSmaReq = {0};
+  int32_t code = buildCreateSmaReq(pCxt, pStmt, &createSmaReq);
   if (TSDB_CODE_SUCCESS != code) {
     return code;
   }
@@ -1445,14 +1536,13 @@ static int32_t translateCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt
   }
   pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet;
   pCxt->pCmdMsg->msgType = TDMT_VND_CREATE_SMA;
-  pCxt->pCmdMsg->msgLen = tSerializeSVCreateTSmaReq(NULL, &createSmaReq);
+  pCxt->pCmdMsg->msgLen = tSerializeSMCreateSmaReq(NULL, 0, &createSmaReq);
   pCxt->pCmdMsg->pMsg = malloc(pCxt->pCmdMsg->msgLen);
   if (NULL == pCxt->pCmdMsg->pMsg) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
-  void* pBuf = pCxt->pCmdMsg->pMsg;
-  tSerializeSVCreateTSmaReq(&pBuf, &createSmaReq);
-  tdDestroyTSma(&createSmaReq.tSma);
+  tSerializeSMCreateSmaReq(pCxt->pCmdMsg->pMsg, pCxt->pCmdMsg->msgLen, &createSmaReq);
+  tFreeSMCreateSmaReq(&createSmaReq);
 
   return TSDB_CODE_SUCCESS;
 }

source/libs/qcom/src/queryUtil.c

@@ -140,7 +140,7 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code)
   return 0;
 }
 
-int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo) {
+int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo, bool persistHandle, void *rpcCtx) {
   char* pMsg = rpcMallocCont(pInfo->msgInfo.len);
   if (NULL == pMsg) {
     qError("0x%" PRIx64 " msg:%s malloc failed", pInfo->requestId, TMSG_INFO(pInfo->msgType));
@@ -154,6 +154,7 @@ int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransp
                     .contLen = pInfo->msgInfo.len,
                     .ahandle = (void*)pInfo,
                     .handle = pInfo->msgInfo.handle,
+                    .persistHandle = persistHandle,
                     .code = 0};
   if (pInfo->msgType == TDMT_VND_QUERY || pInfo->msgType == TDMT_VND_FETCH ||
       pInfo->msgType == TDMT_VND_QUERY_CONTINUE) {
@@ -162,10 +163,14 @@ int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransp
 
   assert(pInfo->fp != NULL);
 
-  rpcSendRequest(pTransporter, epSet, &rpcMsg, pTransporterId);
+  rpcSendRequestWithCtx(pTransporter, epSet, &rpcMsg, pTransporterId, rpcCtx);
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo) {
+  return asyncSendMsgToServerExt(pTransporter, epSet, pTransporterId, pInfo, false, NULL);
+}
+
 char *jobTaskStatusStr(int32_t status) {
   switch (status) {
     case JOB_TASK_STATUS_NULL:

source/libs/qworker/inc/qworkerInt.h

@@ -69,18 +69,24 @@ enum {
 typedef struct SQWDebug {
   bool lockEnable;
   bool statusEnable;
+  bool dumpEnable;
 } SQWDebug;
 
+typedef struct SQWConnInfo {
+  void *handle;
+  void *ahandle;
+} SQWConnInfo;
+
 typedef struct SQWMsg {
-  void   *node;
-  char   *msg;
-  int32_t msgLen;
-  void   *connection;
+  void        *node;
+  char        *msg;
+  int32_t      msgLen;
+  SQWConnInfo  connInfo;
 } SQWMsg;
 
 typedef struct SQWHbInfo {
   SSchedulerHbRsp  rsp;
-  void            *connection;
+  SQWConnInfo      connInfo;
 } SQWHbInfo;
 
 typedef struct SQWPhaseInput {
@@ -101,10 +107,6 @@ typedef struct SQWTaskCtx {
   SRWLatch        lock;
   int8_t          phase;
   int8_t          taskType;
-
-  void           *readyConnection;
-  void           *dropConnection;
-  void           *cancelConnection;
   
   bool            emptyRes;
   bool            queryFetched;
@@ -113,6 +115,7 @@ typedef struct SQWTaskCtx {
   bool            queryInQueue;
   int32_t         rspCode; 
 
+  SQWConnInfo     connInfo;
   int8_t          events[QW_EVENT_MAX];
   
   qTaskInfo_t     taskHandle;
@@ -120,11 +123,12 @@ typedef struct SQWTaskCtx {
 } SQWTaskCtx;
 
 typedef struct SQWSchStatus {
-  int32_t   lastAccessTs; // timestamp in second
-  uint64_t  hbSeqId;
-  void     *hbConnection;
-  SRWLatch  tasksLock;
-  SHashObj *tasksHash;   // key:queryId+taskId, value: SQWTaskStatus
+  int32_t        lastAccessTs; // timestamp in second
+  SRWLatch       hbConnLock;
+  SQWConnInfo    hbConnInfo;
+  SQueryNodeEpId hbEpId;  
+  SRWLatch       tasksLock;
+  SHashObj      *tasksHash;   // key:queryId+taskId, value: SQWTaskStatus
 } SQWSchStatus;
 
 // Qnode/Vnode level task management
@@ -172,12 +176,16 @@ typedef struct SQWorkerMgmt {
 #define QW_ELOG(param, ...) qError("QW:%p " param, mgmt, __VA_ARGS__)
 #define QW_DLOG(param, ...) qDebug("QW:%p " param, mgmt, __VA_ARGS__)
 
+#define QW_DUMP(param, ...) do { if (gQWDebug.dumpEnable) { qDebug("QW:%p " param, mgmt, __VA_ARGS__); } } while (0)
+
+
 #define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__)
 #define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__)
 
 #define QW_TASK_ELOG(param, ...) qError("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
 #define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
 #define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
+#define QW_TASK_DLOGL(param, ...) qDebugL("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__)
 
 #define QW_TASK_ELOG_E(param) qError("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId)
 #define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId)
@@ -223,8 +231,6 @@ typedef struct SQWorkerMgmt {
   }                                                       \
 } while (0)
 
-int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code);
-
 #ifdef __cplusplus
 }
 #endif

source/libs/qworker/inc/qworkerMsg.h

@@ -30,17 +30,18 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg);
 int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg);
 int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req);
 
-int32_t qwBuildAndSendDropRsp(void *connection, int32_t code);
-int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code);
-int32_t qwBuildAndSendFetchRsp(void *connection, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code);
+int32_t qwBuildAndSendDropRsp(SQWConnInfo *pConn, int32_t code);
+int32_t qwBuildAndSendCancelRsp(SQWConnInfo *pConn, int32_t code);
+int32_t qwBuildAndSendFetchRsp(SQWConnInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code);
 void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete);
-int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, void *connection);
-int32_t qwBuildAndSendReadyRsp(void *connection, int32_t code);
-int32_t qwBuildAndSendQueryRsp(void *connection, int32_t code);
+int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SQWConnInfo *pConn);
+int32_t qwBuildAndSendReadyRsp(SQWConnInfo *pConn, int32_t code);
+int32_t qwBuildAndSendQueryRsp(SQWConnInfo *pConn, int32_t code);
 void qwFreeFetchRsp(void *msg);
 int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp);
 int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRsp **rsp);
-int32_t qwBuildAndSendHbRsp(SRpcMsg *pMsg, SSchedulerHbRsp *rsp, int32_t code);
+int32_t qwBuildAndSendHbRsp(SQWConnInfo *pConn, SSchedulerHbRsp *rsp, int32_t code);
+int32_t qwRegisterBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn);
 
 
 

source/libs/qworker/src/qworker.c

@@ -9,7 +9,7 @@
 #include "tname.h"
 #include "dataSinkMgt.h"
 
-SQWDebug gQWDebug = {.statusEnable = true};
+SQWDebug gQWDebug = {.statusEnable = true, .dumpEnable = true};
 
 int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore) {
   if (!gQWDebug.statusEnable) {
@@ -103,6 +103,36 @@ _return:
   QW_RET(code);
 }
 
+void qwDbgDumpSchInfo(SQWSchStatus *sch, int32_t i) {
+
+}
+
+void qwDbgDumpMgmtInfo(SQWorkerMgmt *mgmt) {
+  if (!gQWDebug.dumpEnable) {
+    return;
+  }
+
+  QW_LOCK(QW_READ, &mgmt->schLock);
+  
+  QW_DUMP("total remain schduler num:%d", taosHashGetSize(mgmt->schHash));
+
+  void *key = NULL;
+  size_t keyLen = 0;
+  int32_t i = 0;
+  SQWSchStatus *sch = NULL;
+
+  void *pIter = taosHashIterate(mgmt->schHash, NULL);
+  while (pIter) {
+    sch = (SQWSchStatus *)pIter;
+    qwDbgDumpSchInfo(sch, i);
+    ++i;
+    pIter = taosHashIterate(mgmt->schHash, pIter);
+  }
+
+  QW_UNLOCK(QW_READ, &mgmt->schLock);
+
+  QW_DUMP("total remain ctx num:%d", taosHashGetSize(mgmt->ctxHash));
+}
 
 char *qwPhaseStr(int32_t phase) {
   switch (phase) {
@@ -166,7 +196,7 @@ int32_t qwSetTaskStatus(QW_FPARAMS_DEF, SQWTaskStatus *task, int8_t status) {
 }
 
 
-int32_t qwAddSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch) {
+int32_t qwAddSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType) {
   SQWSchStatus newSch = {0};
   newSch.tasksHash = taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
   if (NULL == newSch.tasksHash) {
@@ -200,7 +230,7 @@ int32_t qwAcquireSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType,
       QW_UNLOCK(rwType, &mgmt->schLock);
       
       if (QW_NOT_EXIST_ADD == nOpt) {
-        QW_ERR_RET(qwAddSchedulerImpl(mgmt, sId, rwType, sch));
+        QW_ERR_RET(qwAddSchedulerImpl(mgmt, sId, rwType));
 
         nOpt = QW_NOT_EXIST_RET_ERR;
         
@@ -402,6 +432,9 @@ int32_t qwKillTaskHandle(QW_FPARAMS_DEF, SQWTaskCtx *ctx) {
 
 
 void qwFreeTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) {
+  rpcReleaseHandle(ctx->connInfo.handle, TAOS_CONN_SERVER);
+  ctx->connInfo.handle = NULL;
+
   qwFreeTaskHandle(QW_FPARAMS(), &ctx->taskHandle);
   
   if (ctx->sinkHandle) {
@@ -516,7 +549,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) {
  
   while (true) {
     QW_TASK_DLOG("start to execTask, loopIdx:%d", i++);
-    
+
     code = qExecTask(*taskHandle, &pRes, &useconds);
     if (code) {
       QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code));
@@ -577,6 +610,9 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) {
 int32_t qwGenerateSchHbRsp(SQWorkerMgmt *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) {
   int32_t taskNum = 0;
 
+  hbInfo->connInfo = sch->hbConnInfo;
+  hbInfo->rsp.epId = sch->hbEpId;
+
   QW_LOCK(QW_READ, &sch->tasksLock);
   
   taskNum = taosHashGetSize(sch->tasksHash);
@@ -588,9 +624,6 @@ int32_t qwGenerateSchHbRsp(SQWorkerMgmt *mgmt, SQWSchStatus *sch, SQWHbInfo *hbI
     return TSDB_CODE_QRY_OUT_OF_MEMORY;
   }
 
-  hbInfo->connection = sch->hbConnection;
-  hbInfo->rsp.seqId = -1;
-
   void *key = NULL;
   size_t keyLen = 0;
   int32_t i = 0;
@@ -694,8 +727,8 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void
 int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) {
   int32_t code = 0;
   SQWTaskCtx *ctx = NULL;
-  void *dropConnection = NULL;
-  void *cancelConnection = NULL;
+  SQWConnInfo *dropConnection = NULL;
+  SQWConnInfo *cancelConnection = NULL;
 
   QW_TASK_DLOG("start to handle event at phase %s", qwPhaseStr(phase));
 
@@ -727,9 +760,13 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu
       }
 
       if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) {
+        dropConnection = &ctx->connInfo;
         QW_ERR_JRET(qwDropTask(QW_FPARAMS()));
+        dropConnection = NULL;
+        
+        qwBuildAndSendDropRsp(&ctx->connInfo, code);    
+        QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->connInfo.handle, code, tstrerror(code));
 
-        dropConnection = ctx->dropConnection;
         QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED);
         break;
       }
@@ -761,9 +798,13 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu
       }
 
       if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) {
+        dropConnection = &ctx->connInfo;
         QW_ERR_JRET(qwDropTask(QW_FPARAMS()));
+        dropConnection = NULL;
 
-        dropConnection = ctx->dropConnection;
+        qwBuildAndSendDropRsp(&ctx->connInfo, code);
+        QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->connInfo.handle, code, tstrerror(code));
+        
         QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED);
       }
 
@@ -789,13 +830,13 @@ _return:
   }
 
   if (dropConnection) {
-    qwBuildAndSendDropRsp(dropConnection, code);
-    QW_TASK_DLOG("drop msg rsped, code:%x - %s", code, tstrerror(code));
+    qwBuildAndSendDropRsp(dropConnection, code);    
+    QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", dropConnection->handle, code, tstrerror(code));
   }
 
   if (cancelConnection) {
-    qwBuildAndSendCancelRsp(cancelConnection, code);
-    QW_TASK_DLOG("cancel msg rsped, code:%x - %s", code, tstrerror(code));
+    qwBuildAndSendCancelRsp(cancelConnection, code);    
+    QW_TASK_DLOG("cancel rsp send, handle:%p, code:%x - %s", cancelConnection->handle, code, tstrerror(code));
   }
 
   QW_TASK_DLOG("end to handle event at phase %s, code:%x - %s", qwPhaseStr(phase), code, tstrerror(code));
@@ -807,9 +848,8 @@ _return:
 int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) {
   int32_t code = 0;
   SQWTaskCtx *ctx = NULL;
-  void *readyConnection = NULL;
-  void *dropConnection = NULL;
-  void *cancelConnection = NULL;
+  SQWConnInfo connInfo = {0};
+  SQWConnInfo *readyConnection = NULL;
 
   QW_TASK_DLOG("start to handle event at phase %s", qwPhaseStr(phase));
   
@@ -826,11 +866,18 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp
     if (NULL == ctx->taskHandle && NULL == ctx->sinkHandle) {
       ctx->emptyRes = true;
     }
-    
+
+#if 0    
     if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_READY)) {
-      readyConnection = ctx->readyConnection;
+      readyConnection = &ctx->connInfo;
       QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY);
     }
+#else
+    connInfo.handle = ctx->connInfo.handle;
+    readyConnection = &connInfo;
+    
+    QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY);
+#endif
   }
 
   if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) {
@@ -838,10 +885,11 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp
       QW_TASK_WLOG("drop received at wrong phase %s", qwPhaseStr(phase));
       QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
     }
+
+    qwBuildAndSendDropRsp(&ctx->connInfo, code);    
+    QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->connInfo.handle, code, tstrerror(code));
     
     QW_ERR_JRET(qwDropTask(QW_FPARAMS()));
-
-    dropConnection = ctx->dropConnection;
     QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED);
   }
 
@@ -869,19 +917,9 @@ _return:
     qwReleaseTaskCtx(mgmt, ctx);
   }
 
-  if (readyConnection) {
-    qwBuildAndSendReadyRsp(readyConnection, code);
-    QW_TASK_DLOG("ready msg rsped, code:%x - %s", code, tstrerror(code));
-  }
-
-  if (dropConnection) {
-    qwBuildAndSendDropRsp(dropConnection, code);
-    QW_TASK_DLOG("drop msg rsped, code:%x - %s", code, tstrerror(code));
-  }
-
-  if (cancelConnection) {
-    qwBuildAndSendCancelRsp(cancelConnection, code);
-    QW_TASK_DLOG("cancel msg rsped, code:%x - %s", code, tstrerror(code));
+  if (TSDB_CODE_SUCCESS == code && readyConnection) {
+    qwBuildAndSendReadyRsp(readyConnection, code);    
+    QW_TASK_DLOG("ready msg rsped, handle:%p, code:%x - %s", readyConnection->handle, code, tstrerror(code));
   }
 
   if (code) {
@@ -893,23 +931,26 @@ _return:
   QW_RET(code);
 }
 
-
 int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) {
   int32_t code = 0;
   bool queryRsped = false;
-  bool needStop = false;
   struct SSubplan *plan = NULL;
   SQWPhaseInput input = {0};
   qTaskInfo_t pTaskInfo = NULL;
   DataSinkHandle sinkHandle = NULL;
   SQWTaskCtx *ctx = NULL;
 
+  QW_ERR_JRET(qwRegisterBrokenLinkArg(QW_FPARAMS(), &qwMsg->connInfo));
+
   QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_QUERY, &input, NULL));
 
   QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx));
   
   atomic_store_8(&ctx->taskType, taskType);
-  
+  atomic_store_ptr(&ctx->connInfo.handle, qwMsg->connInfo.handle);
+  atomic_store_ptr(&ctx->connInfo.ahandle, qwMsg->connInfo.ahandle);
+
+  QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg);
   code = qStringToSubplan(qwMsg->msg, &plan);
   if (TSDB_CODE_SUCCESS != code) {
     QW_TASK_ELOG("task string to subplan failed, code:%x - %s", code, tstrerror(code));
@@ -927,8 +968,8 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) {
     QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
   }
 
-  QW_ERR_JRET(qwBuildAndSendQueryRsp(qwMsg->connection, code));
-  QW_TASK_DLOG("query msg rsped, code:%x - %s", code, tstrerror(code));
+  QW_ERR_JRET(qwBuildAndSendQueryRsp(&qwMsg->connInfo, code));
+  QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));
 
   queryRsped = true;
 
@@ -945,11 +986,11 @@ _return:
   code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL);
   
   if (!queryRsped) {
-    qwBuildAndSendQueryRsp(qwMsg->connection, code);
-    QW_TASK_DLOG("query msg rsped, code:%x - %s", code, tstrerror(code));
+    qwBuildAndSendQueryRsp(&qwMsg->connInfo, code);
+    QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));
   }
 
-  QW_RET(code);
+  QW_RET(TSDB_CODE_SUCCESS);
 }
 
 int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
@@ -968,8 +1009,9 @@ int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
   }
   
   if (ctx->phase == QW_PHASE_PRE_QUERY) {
+    ctx->connInfo.handle == qwMsg->connInfo.handle;
+    ctx->connInfo.ahandle = qwMsg->connInfo.ahandle;
     QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_READY);
-    ctx->readyConnection = qwMsg->connection;
     needRsp = false;
     QW_TASK_DLOG_E("ready msg will not rsp now");
     goto _return;
@@ -1007,11 +1049,11 @@ _return:
   }
 
   if (needRsp) {
-    qwBuildAndSendReadyRsp(qwMsg->connection, code);
-    QW_TASK_DLOG("ready msg rsped, code:%x - %s", code, tstrerror(code));
+    qwBuildAndSendReadyRsp(&qwMsg->connInfo, code);
+    QW_TASK_DLOG("ready msg rsped, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));
   }
 
-  QW_RET(code);
+  QW_RET(TSDB_CODE_SUCCESS);
 }
 
 
@@ -1048,10 +1090,11 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
         qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete);
         atomic_store_8((int8_t*)&ctx->queryEnd, qComplete);
 
+        qwMsg->connInfo = ctx->connInfo;
         QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH);            
         
-        qwBuildAndSendFetchRsp(qwMsg->connection, rsp, dataLen, code);                
-        QW_TASK_DLOG("fetch msg rsped, code:%x, dataLen:%d", code, dataLen);
+        qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, dataLen, code);                
+        QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), dataLen);
       } else {
         atomic_store_8((int8_t*)&ctx->queryContinue, 1);
       }
@@ -1067,8 +1110,10 @@ _return:
       QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH);    
       qwFreeFetchRsp(rsp);
       rsp = NULL;
-      qwBuildAndSendFetchRsp(qwMsg->connection, rsp, 0, code);
-      QW_TASK_DLOG("fetch msg rsped, code:%x - %s", code, tstrerror(code));
+      
+      qwMsg->connInfo = ctx->connInfo;
+      qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, 0, code);
+      QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), 0);
     }
 
     QW_LOCK(QW_WRITE, &ctx->lock);
@@ -1082,7 +1127,9 @@ _return:
   } while (true);
 
   input.code = code;
-  QW_RET(qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_CQUERY, &input, NULL));
+  qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_CQUERY, &input, NULL);    
+
+  QW_RET(TSDB_CODE_SUCCESS);  
 }
 
 
@@ -1097,11 +1144,14 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
   QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_FETCH, &input, NULL));
 
   QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx));
- 
+  
   SOutputData sOutput = {0};
   QW_ERR_JRET(qwGetResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput));
 
   if (NULL == rsp) {
+    atomic_store_ptr(&ctx->connInfo.handle, qwMsg->connInfo.handle);
+    atomic_store_ptr(&ctx->connInfo.ahandle, qwMsg->connInfo.ahandle);
+    
     QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_FETCH);
   } else {
     bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd);
@@ -1123,7 +1173,7 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
 
       atomic_store_8((int8_t*)&ctx->queryInQueue, 1);
       
-      QW_ERR_JRET(qwBuildAndSendCQueryMsg(QW_FPARAMS(), qwMsg->connection));
+      QW_ERR_JRET(qwBuildAndSendCQueryMsg(QW_FPARAMS(), &qwMsg->connInfo));
     }
   }
   
@@ -1143,17 +1193,17 @@ _return:
   }
 
   if (code || rsp) {
-    qwBuildAndSendFetchRsp(qwMsg->connection, rsp, dataLen, code);
-    QW_TASK_DLOG("fetch msg rsped, code:%x, dataLen:%d", code, dataLen);
+    qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, dataLen, code);
+    QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), dataLen);
   }
 
-  QW_RET(code);
+  QW_RET(TSDB_CODE_SUCCESS);
 }
 
 
 int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
   int32_t code = 0;
-  bool needRsp = false;
+  bool rsped = false;
   SQWTaskCtx *ctx = NULL;
   bool locked = false;
 
@@ -1174,14 +1224,18 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
     QW_ERR_JRET(qwKillTaskHandle(QW_FPARAMS(), ctx));
     qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROPPING);
   } else if (ctx->phase > 0) {
+    qwBuildAndSendDropRsp(&qwMsg->connInfo, code);
+    QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));
+  
     QW_ERR_JRET(qwDropTask(QW_FPARAMS()));
-    needRsp = true;
+    rsped = true;
   } else {
     // task not started
   }
 
-  if (!needRsp) {    
-    ctx->dropConnection = qwMsg->connection;
+  if (!rsped) {
+    ctx->connInfo.handle = qwMsg->connInfo.handle;
+    ctx->connInfo.ahandle = qwMsg->connInfo.ahandle;
     
     QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP);
   }
@@ -1204,13 +1258,12 @@ _return:
     qwReleaseTaskCtx(mgmt, ctx);
   }
 
-  if (TSDB_CODE_SUCCESS != code || needRsp) {
-    QW_ERR_RET(qwBuildAndSendDropRsp(qwMsg->connection, code));
-
-    QW_TASK_DLOG("drop msg rsped, code:%x", code);
+  if (TSDB_CODE_SUCCESS != code) {
+    qwBuildAndSendDropRsp(&qwMsg->connInfo, code);
+    QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));
   }
 
-  QW_RET(code);
+  QW_RET(TSDB_CODE_SUCCESS);
 }
 
 int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) {
@@ -1218,38 +1271,46 @@ int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) {
   SSchedulerHbRsp rsp = {0};
   SQWSchStatus *sch = NULL;
   uint64_t seqId = 0;
+  void *origHandle = NULL;
 
   memcpy(&rsp.epId, &req->epId, sizeof(req->epId));
   
   QW_ERR_JRET(qwAcquireAddScheduler(mgmt, req->sId, QW_READ, &sch));
 
-  atomic_store_ptr(&sch->hbConnection, qwMsg->connection);
-  ++sch->hbSeqId;
+  QW_LOCK(QW_WRITE, &sch->hbConnLock);
 
-  rsp.seqId = sch->hbSeqId;
-
-  QW_DLOG("hb connection updated, seqId:%" PRIx64 ", sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, connection:%p",
-    sch->hbSeqId, req->sId, req->epId.nodeId, req->epId.ep.fqdn, req->epId.ep.port, qwMsg->connection);
+  if (sch->hbConnInfo.handle) {
+    rpcReleaseHandle(sch->hbConnInfo.handle, TAOS_CONN_SERVER);
+  }
+  
+  memcpy(&sch->hbConnInfo, &qwMsg->connInfo, sizeof(qwMsg->connInfo));
+  memcpy(&sch->hbEpId, &req->epId, sizeof(req->epId));
+  
+  QW_UNLOCK(QW_WRITE, &sch->hbConnLock);
+  
+  QW_DLOG("hb connection updated, sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, handle:%p, ahandle:%p",
+    req->sId, req->epId.nodeId, req->epId.ep.fqdn, req->epId.ep.port, qwMsg->connInfo.handle, qwMsg->connInfo.ahandle);
 
   qwReleaseScheduler(QW_READ, mgmt);
 
 _return:
 
-  qwBuildAndSendHbRsp(qwMsg->connection, &rsp, code);
+  qwBuildAndSendHbRsp(&qwMsg->connInfo, &rsp, code);
+  QW_DLOG("hb rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));
   
-  QW_RET(code);
+  QW_RET(TSDB_CODE_SUCCESS);
 }
 
 
 void qwProcessHbTimerEvent(void *param, void *tmrId) {
-  return;
-  
   SQWorkerMgmt *mgmt = (SQWorkerMgmt *)param;
   SQWSchStatus *sch = NULL;
   int32_t taskNum = 0;
   SQWHbInfo *rspList = NULL;
   int32_t code = 0;
 
+  qwDbgDumpMgmtInfo(mgmt);
+
   QW_LOCK(QW_READ, &mgmt->schLock);
 
   int32_t schNum = taosHashGetSize(mgmt->schHash);
@@ -1288,8 +1349,8 @@ _return:
   QW_UNLOCK(QW_READ, &mgmt->schLock);
 
   for (int32_t j = 0; j < i; ++j) {
-    QW_DLOG("hb on connection %p, taskNum:%d", rspList[j].connection, (rspList[j].rsp.taskStatus ? (int32_t)taosArrayGetSize(rspList[j].rsp.taskStatus) : 0));
-    qwBuildAndSendHbRsp(rspList[j].connection, &rspList[j].rsp, code);
+    qwBuildAndSendHbRsp(&rspList[j].connInfo, &rspList[j].rsp, code);
+    QW_DLOG("hb rsp send, handle:%p, code:%x - %s, taskNum:%d", rspList[j].connInfo.handle, code, tstrerror(code), (rspList[j].rsp.taskStatus ? (int32_t)taosArrayGetSize(rspList[j].rsp.taskStatus) : 0));    
     tFreeSSchedulerHbRsp(&rspList[j].rsp);
   }
 

source/libs/qworker/src/qworkerMsg.c

@@ -26,6 +26,8 @@ int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp) {
   return TSDB_CODE_SUCCESS;
 }
 
+
+
 void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete) {
   SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg;
   
@@ -44,8 +46,7 @@ void qwFreeFetchRsp(void *msg) {
   }
 }
 
-int32_t qwBuildAndSendQueryRsp(void *connection, int32_t code) {
-  SRpcMsg *pMsg = (SRpcMsg *)connection;
+int32_t qwBuildAndSendQueryRsp(SQWConnInfo *pConn, int32_t code) {
   SQueryTableRsp rsp = {.code = code};
   
   int32_t contLen = tSerializeSQueryTableRsp(NULL, 0, &rsp);
@@ -54,8 +55,8 @@ int32_t qwBuildAndSendQueryRsp(void *connection, int32_t code) {
 
   SRpcMsg rpcRsp = {
     .msgType = TDMT_VND_QUERY_RSP,
-    .handle  = pMsg->handle,
-    .ahandle = pMsg->ahandle,
+    .handle  = pConn->handle,
+    .ahandle = pConn->ahandle,
     .pCont   = msg,
     .contLen = contLen,
     .code    = code,
@@ -66,15 +67,14 @@ int32_t qwBuildAndSendQueryRsp(void *connection, int32_t code) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t qwBuildAndSendReadyRsp(void *connection, int32_t code) {
-  SRpcMsg *pMsg = (SRpcMsg *)connection;
+int32_t qwBuildAndSendReadyRsp(SQWConnInfo *pConn, int32_t code) {
   SResReadyRsp *pRsp = (SResReadyRsp *)rpcMallocCont(sizeof(SResReadyRsp));
   pRsp->code = code;
 
   SRpcMsg rpcRsp = {
     .msgType = TDMT_VND_RES_READY_RSP,
-    .handle  = pMsg->handle,
-    .ahandle = pMsg->ahandle,
+    .handle  = pConn->handle,
+    .ahandle = NULL,
     .pCont   = pRsp,
     .contLen = sizeof(*pRsp),
     .code    = code,
@@ -85,15 +85,15 @@ int32_t qwBuildAndSendReadyRsp(void *connection, int32_t code) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t qwBuildAndSendHbRsp(SRpcMsg *pMsg, SSchedulerHbRsp *pStatus, int32_t code) {
+int32_t qwBuildAndSendHbRsp(SQWConnInfo *pConn, SSchedulerHbRsp *pStatus, int32_t code) {
   int32_t contLen = tSerializeSSchedulerHbRsp(NULL, 0, pStatus);
   void *pRsp = rpcMallocCont(contLen);
   tSerializeSSchedulerHbRsp(pRsp, contLen, pStatus);
 
   SRpcMsg rpcRsp = {
     .msgType = TDMT_VND_QUERY_HEARTBEAT_RSP,
-    .handle  = pMsg->handle,
-    .ahandle = pMsg->ahandle,
+    .handle  = pConn->handle,
+    .ahandle = pConn->ahandle,
     .pCont   = pRsp,
     .contLen = contLen,
     .code    = code,
@@ -104,9 +104,7 @@ int32_t qwBuildAndSendHbRsp(SRpcMsg *pMsg, SSchedulerHbRsp *pStatus, int32_t cod
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t qwBuildAndSendFetchRsp(void *connection, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code) {
-  SRpcMsg *pMsg = (SRpcMsg *)connection;
-  
+int32_t qwBuildAndSendFetchRsp(SQWConnInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code) {
   if (NULL == pRsp) {
     pRsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
     memset(pRsp, 0, sizeof(SRetrieveTableRsp));
@@ -115,8 +113,8 @@ int32_t qwBuildAndSendFetchRsp(void *connection, SRetrieveTableRsp *pRsp, int32_
 
   SRpcMsg rpcRsp = {
     .msgType = TDMT_VND_FETCH_RSP,
-    .handle  = pMsg->handle,
-    .ahandle = pMsg->ahandle,
+    .handle  = pConn->handle,
+    .ahandle = pConn->ahandle,
     .pCont   = pRsp,
     .contLen = sizeof(*pRsp) + dataLength,
     .code    = code,
@@ -127,14 +125,14 @@ int32_t qwBuildAndSendFetchRsp(void *connection, SRetrieveTableRsp *pRsp, int32_
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code) {
+int32_t qwBuildAndSendCancelRsp(SQWConnInfo *pConn, int32_t code) {
   STaskCancelRsp *pRsp = (STaskCancelRsp *)rpcMallocCont(sizeof(STaskCancelRsp));
   pRsp->code = code;
 
   SRpcMsg rpcRsp = {
     .msgType = TDMT_VND_CANCEL_TASK_RSP,
-    .handle  = pMsg->handle,
-    .ahandle = pMsg->ahandle,
+    .handle  = pConn->handle,
+    .ahandle = pConn->ahandle,
     .pCont   = pRsp,
     .contLen = sizeof(*pRsp),
     .code    = code,
@@ -144,15 +142,14 @@ int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t qwBuildAndSendDropRsp(void *connection, int32_t code) {
-  SRpcMsg *pMsg = (SRpcMsg *)connection;
+int32_t qwBuildAndSendDropRsp(SQWConnInfo *pConn, int32_t code) {
   STaskDropRsp *pRsp = (STaskDropRsp *)rpcMallocCont(sizeof(STaskDropRsp));
   pRsp->code = code;
 
   SRpcMsg rpcRsp = {
     .msgType = TDMT_VND_DROP_TASK_RSP,
-    .handle  = pMsg->handle,
-    .ahandle = pMsg->ahandle,
+    .handle  = pConn->handle,
+    .ahandle = pConn->ahandle,
     .pCont   = pRsp,
     .contLen = sizeof(*pRsp),
     .code    = code,
@@ -234,8 +231,7 @@ int32_t qwBuildAndSendShowFetchRsp(SRpcMsg *pMsg, SVShowTablesFetchReq* pFetchRe
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, void *connection) {
-  SRpcMsg *pMsg = (SRpcMsg *)connection;
+int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SQWConnInfo *pConn) {
   SQueryContinueReq * req = (SQueryContinueReq *)rpcMallocCont(sizeof(SQueryContinueReq));
   if (NULL == req) {
     QW_SCH_TASK_ELOG("rpcMallocCont %d failed", (int32_t)sizeof(SQueryContinueReq));
@@ -248,12 +244,12 @@ int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, void *connection) {
   req->taskId = tId;
 
   SRpcMsg pNewMsg = {
-      .handle = pMsg->handle,
-      .ahandle = pMsg->ahandle,
-      .msgType = TDMT_VND_QUERY_CONTINUE,
-      .pCont = req,
-      .contLen = sizeof(SQueryContinueReq),
-      .code = 0,
+    .handle  = pConn->handle,
+    .ahandle = pConn->ahandle,
+    .msgType = TDMT_VND_QUERY_CONTINUE,
+    .pCont   = req,
+    .contLen = sizeof(SQueryContinueReq),
+    .code    = 0,
   };
 
   int32_t code = tmsgPutToQueue(&mgmt->msgCb, QUERY_QUEUE, &pNewMsg);
@@ -268,6 +264,35 @@ int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, void *connection) {
   return TSDB_CODE_SUCCESS;
 }
 
+
+int32_t qwRegisterBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn) {
+  STaskDropReq * req = (STaskDropReq *)rpcMallocCont(sizeof(STaskDropReq));
+  if (NULL == req) {
+    QW_SCH_TASK_ELOG("rpcMallocCont %d failed", (int32_t)sizeof(STaskDropReq));
+    QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  }  
+
+  req->header.vgId = htonl(mgmt->nodeId);
+  req->sId = htobe64(sId);
+  req->queryId = htobe64(qId);
+  req->taskId = htobe64(tId);
+  req->refId = htobe64(rId);
+  
+  SRpcMsg pMsg = {
+    .handle  = pConn->handle,
+    .ahandle = pConn->ahandle,
+    .msgType = TDMT_VND_DROP_TASK,
+    .pCont   = req,
+    .contLen = sizeof(STaskDropReq),
+    .code    = TSDB_CODE_RPC_NETWORK_UNAVAIL,
+  };
+  
+  rpcRegisterBrokenLinkArg(&pMsg);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
 int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
   if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) {
     QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
@@ -294,10 +319,12 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
   uint64_t tId = msg->taskId;
   int64_t  rId = msg->refId;
 
-  SQWMsg qwMsg = {.node = node, .msg = msg->msg + msg->sqlLen, .msgLen = msg->phyLen, .connection = pMsg};
+  SQWMsg qwMsg = {.node = node, .msg = msg->msg + msg->sqlLen, .msgLen = msg->phyLen};
+  qwMsg.connInfo.handle = pMsg->handle;
+  qwMsg.connInfo.ahandle = pMsg->ahandle;
 
   char* sql = strndup(msg->msg, msg->sqlLen);
-  QW_SCH_TASK_DLOG("processQuery start, node:%p, sql:%s", node, sql);
+  QW_SCH_TASK_DLOG("processQuery start, node:%p, handle:%p, sql:%s", node, pMsg->handle, sql);
   tfree(sql);
 
   QW_ERR_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg, msg->taskType));
@@ -326,9 +353,11 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
   uint64_t tId = msg->taskId;
   int64_t rId = 0;
 
-  SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .connection = pMsg};
+  SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0};
+  qwMsg.connInfo.handle = pMsg->handle;
+  qwMsg.connInfo.ahandle = pMsg->ahandle;
 
-  QW_SCH_TASK_DLOG("processCQuery start, node:%p", node);
+  QW_SCH_TASK_DLOG("processCQuery start, node:%p, handle:%p", node, pMsg->handle);
 
   QW_ERR_RET(qwProcessCQuery(QW_FPARAMS(), &qwMsg));
 
@@ -358,9 +387,11 @@ int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){
   uint64_t tId = msg->taskId;
   int64_t rId = 0;
 
-  SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .connection = pMsg};
+  SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0};
+  qwMsg.connInfo.handle = pMsg->handle;
+  qwMsg.connInfo.ahandle = pMsg->ahandle;
 
-  QW_SCH_TASK_DLOG("processReady start, node:%p", node);
+  QW_SCH_TASK_DLOG("processReady start, node:%p, handle:%p", node, pMsg->handle);
 
   QW_ERR_RET(qwProcessReady(QW_FPARAMS(), &qwMsg));
 
@@ -418,9 +449,11 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
   uint64_t tId = msg->taskId;
   int64_t rId = 0;
 
-  SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .connection = pMsg};
+  SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0};
+  qwMsg.connInfo.handle = pMsg->handle;
+  qwMsg.connInfo.ahandle = pMsg->ahandle;
 
-  QW_SCH_TASK_DLOG("processFetch start, node:%p", node);
+  QW_SCH_TASK_DLOG("processFetch start, node:%p, handle:%p", node, pMsg->handle);
 
   QW_ERR_RET(qwProcessFetch(QW_FPARAMS(), &qwMsg));
 
@@ -439,6 +472,7 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
     return TSDB_CODE_QRY_INVALID_INPUT;
   }
 
+  SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt;
   int32_t code = 0;
   STaskCancelReq *msg = pMsg->pCont;
   if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
@@ -451,11 +485,21 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
   msg->taskId = be64toh(msg->taskId);
   msg->refId = be64toh(msg->refId);
 
+  uint64_t sId = msg->sId;
+  uint64_t qId = msg->queryId;
+  uint64_t tId = msg->taskId;
+  int64_t  rId = msg->refId;
+
+  SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0};
+  qwMsg.connInfo.handle = pMsg->handle;
+  qwMsg.connInfo.ahandle = pMsg->ahandle;
+
   //QW_ERR_JRET(qwCancelTask(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId));
 
 _return:
 
-  QW_ERR_RET(qwBuildAndSendCancelRsp(pMsg, code));
+  QW_ERR_RET(qwBuildAndSendCancelRsp(&qwMsg.connInfo, code));
+  QW_SCH_TASK_DLOG("cancel rsp send, handle:%p, code:%x - %s", qwMsg.connInfo.handle, code, tstrerror(code));
 
   return TSDB_CODE_SUCCESS;
 }
@@ -484,9 +528,15 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
   uint64_t tId = msg->taskId;
   int64_t  rId = msg->refId;
 
-  SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .connection = pMsg};
+  SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0};
+  qwMsg.connInfo.handle = pMsg->handle;
+  qwMsg.connInfo.ahandle = pMsg->ahandle;
+
+  if (TSDB_CODE_RPC_NETWORK_UNAVAIL == pMsg->code) {
+    QW_SCH_TASK_DLOG("receive drop task due to network broken, error:%s", tstrerror(pMsg->code));    
+  }
 
-  QW_SCH_TASK_DLOG("processDrop start, node:%p", node);
+  QW_SCH_TASK_DLOG("processDrop start, node:%p, handle:%p", node, pMsg->handle);
 
   QW_ERR_RET(qwProcessDrop(QW_FPARAMS(), &qwMsg));
 
@@ -516,9 +566,11 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
   }
 
   uint64_t sId = req.sId;
-  SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .connection = pMsg};
+  SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0};
+  qwMsg.connInfo.handle = pMsg->handle;
+  qwMsg.connInfo.ahandle = pMsg->ahandle;
 
-  QW_SCH_DLOG("processHb start, node:%p", node);
+  QW_SCH_DLOG("processHb start, node:%p, handle:%p", node, pMsg->handle);
 
   QW_ERR_RET(qwProcessHb(mgmt, &qwMsg, &req));
 
@@ -535,7 +587,7 @@ int32_t qWorkerProcessShowMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
 
   int32_t code = 0;
   SVShowTablesReq *pReq = pMsg->pCont;
-  QW_ERR_RET(qwBuildAndSendShowRsp(pMsg, code));
+  QW_RET(qwBuildAndSendShowRsp(pMsg, code));
 }
 
 int32_t qWorkerProcessShowFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
@@ -544,7 +596,7 @@ int32_t qWorkerProcessShowFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg)
   }
 
   SVShowTablesFetchReq *pFetchReq = pMsg->pCont;
-  QW_ERR_RET(qwBuildAndSendShowFetchRsp(pMsg, pFetchReq));
+  QW_RET(qwBuildAndSendShowFetchRsp(pMsg, pFetchReq));
 }
 
 

source/libs/qworker/test/qworkerTests.cpp

@@ -887,8 +887,8 @@ TEST(seqTest, normalCase) {
   code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc);
   ASSERT_EQ(code, 0);
 
-  code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc);
-  ASSERT_EQ(code, 0);
+  //code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc);
+  //ASSERT_EQ(code, 0);
 
   code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc);
   ASSERT_EQ(code, 0);
@@ -985,12 +985,12 @@ TEST(seqTest, randCase) {
       qwtBuildQueryReqMsg(&queryRpc);
       code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc);
     } else if (r >= maxr/5 && r < maxr * 2/5) {
-      printf("Ready,%d\n", t++);
-      qwtBuildReadyReqMsg(&readyMsg, &readyRpc);
-      code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc);
-      if (qwtTestEnableSleep) {
-        taosUsleep(1);
-      }
+      //printf("Ready,%d\n", t++);
+      //qwtBuildReadyReqMsg(&readyMsg, &readyRpc);
+      //code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc);
+      //if (qwtTestEnableSleep) {
+      //  taosUsleep(1);
+      //}
     } else if (r >= maxr * 2/5 && r < maxr* 3/5) {
       printf("Fetch,%d\n", t++);
       qwtBuildFetchReqMsg(&fetchMsg, &fetchRpc);
@@ -1054,7 +1054,7 @@ TEST(seqTest, multithreadRand) {
 
   TdThread t1,t2,t3,t4,t5,t6;
   taosThreadCreate(&(t1), &thattr, queryThread, mgmt);
-  taosThreadCreate(&(t2), &thattr, readyThread, NULL);
+  //taosThreadCreate(&(t2), &thattr, readyThread, NULL);
   taosThreadCreate(&(t3), &thattr, fetchThread, NULL);
   taosThreadCreate(&(t4), &thattr, dropThread, NULL);
   taosThreadCreate(&(t5), &thattr, statusThread, NULL);

source/libs/scheduler/inc/schedulerInt.h

@@ -25,6 +25,7 @@ extern "C" {
 #include "planner.h"
 #include "scheduler.h"
 #include "thash.h"
+#include "trpc.h"
 
 #define SCHEDULE_DEFAULT_MAX_JOB_NUM 1000
 #define SCHEDULE_DEFAULT_MAX_TASK_NUM 1000
@@ -44,7 +45,7 @@ typedef struct SSchTrans {
 
 typedef struct SSchHbTrans {
   SRWLatch  lock;
-  uint64_t  seqId;
+  SRpcCtx   rpcCtx;
   SSchTrans trans;
 } SSchHbTrans;
 
@@ -76,12 +77,23 @@ typedef struct SSchedulerMgmt {
   SHashObj       *hbConnections;
 } SSchedulerMgmt;
 
-typedef struct SSchCallbackParam {
-  uint64_t queryId;
-  int64_t  refId;
-  uint64_t taskId;
-  void    *transport;
-} SSchCallbackParam;
+typedef struct SSchCallbackParamHeader {
+  bool isHbParam;
+} SSchCallbackParamHeader;
+
+typedef struct SSchTaskCallbackParam {
+  SSchCallbackParamHeader head;
+  uint64_t                queryId;
+  int64_t                 refId;
+  uint64_t                taskId;
+  void                   *transport;
+} SSchTaskCallbackParam;
+
+typedef struct SSchHbCallbackParam {
+  SSchCallbackParamHeader head;
+  SQueryNodeEpId          nodeEpId;
+  void                   *transport;
+} SSchHbCallbackParam;
 
 typedef struct SSchFlowControl {
   SRWLatch  lock;
@@ -91,6 +103,11 @@ typedef struct SSchFlowControl {
   SArray   *taskList;      // Element is SSchTask*
 } SSchFlowControl;
 
+typedef struct SSchNodeInfo {
+  SQueryNodeAddr addr;
+  void          *handle;
+} SSchNodeInfo;
+
 typedef struct SSchLevel {
   int32_t         level;
   int8_t          status;
@@ -116,7 +133,7 @@ typedef struct SSchTask {
   SQueryNodeAddr       succeedAddr;    // task executed success node address
   int8_t               candidateIdx;   // current try condidation index
   SArray              *candidateAddrs; // condidate node addresses, element is SQueryNodeAddr
-  SArray              *execAddrs;      // all tried node for current task, element is SQueryNodeAddr
+  SArray              *execNodes;      // all tried node for current task, element is SSchNodeInfo
   SQueryProfileSummary summary;        // task execution summary
   int32_t              childReady;     // child task ready number
   SArray              *children;       // the datasource tasks,from which to fetch the result, element is SQueryTask*
@@ -178,6 +195,8 @@ extern SSchedulerMgmt schMgmt;
 #define SCH_GET_TASK_STATUS(task) atomic_load_8(&(task)->status)
 #define SCH_GET_TASK_STATUS_STR(task) jobTaskStatusStr(SCH_GET_TASK_STATUS(task))
 
+#define SCH_GET_TASK_HANDLE(_task) ((_task) ? (_task)->handle : NULL)
+#define SCH_SET_TASK_HANDLE(_task, _handle) ((_task)->handle = (_handle))
 
 #define SCH_SET_JOB_STATUS(job, st) atomic_store_8(&(job)->status, st)
 #define SCH_GET_JOB_STATUS(job) atomic_load_8(&(job)->status)
@@ -205,6 +224,8 @@ extern SSchedulerMgmt schMgmt;
   qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, pJob->queryId, SCH_TASK_ID(pTask), __VA_ARGS__)
 #define SCH_TASK_DLOG(param, ...) \
   qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, pJob->queryId, SCH_TASK_ID(pTask), __VA_ARGS__)
+#define SCH_TASK_DLOGL(param, ...) \
+  qDebugL("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, pJob->queryId, SCH_TASK_ID(pTask), __VA_ARGS__)
 #define SCH_TASK_WLOG(param, ...) \
   qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, pJob->queryId, SCH_TASK_ID(pTask), __VA_ARGS__)
 
@@ -228,6 +249,8 @@ int32_t schLaunchTasksInFlowCtrlList(SSchJob *pJob, SSchTask *pTask);
 int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask);
 int32_t schFetchFromRemote(SSchJob *pJob);
 int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode);
+int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId);
+int32_t schCloneSMsgSendInfo(void *src, void **dst);
 
 
 #ifdef __cplusplus

source/libs/scheduler/test/schedulerTests.cpp

@@ -387,7 +387,7 @@ void *schtCreateFetchRspThread(void *param) {
 
 void *schtFetchRspThread(void *aa) {
   SDataBuf dataBuf = {0};
-  SSchCallbackParam* param = NULL;
+  SSchTaskCallbackParam* param = NULL;
 
   while (!schtTestStop) {
     if (0 == atomic_val_compare_exchange_32(&schtStartFetch, 1, 0)) {
@@ -396,7 +396,7 @@ void *schtFetchRspThread(void *aa) {
 
     taosUsleep(1);
     
-    param = (SSchCallbackParam *)calloc(1, sizeof(*param));
+    param = (SSchTaskCallbackParam *)calloc(1, sizeof(*param));
 
     param->queryId = schtQueryId;  
     param->taskId = schtFetchTaskId;
@@ -449,7 +449,7 @@ void* schtRunJobThread(void *aa) {
   schtSetAsyncSendMsgToServer();
 
   SSchJob *pJob = NULL;
-  SSchCallbackParam *param = NULL;
+  SSchTaskCallbackParam *param = NULL;
   SHashObj *execTasks = NULL;
   SDataBuf dataBuf = {0};
   uint32_t jobFinished = 0;
@@ -484,7 +484,7 @@ void* schtRunJobThread(void *aa) {
       pIter = taosHashIterate(pJob->execTasks, pIter);
     }    
 
-    param = (SSchCallbackParam *)calloc(1, sizeof(*param));
+    param = (SSchTaskCallbackParam *)calloc(1, sizeof(*param));
     param->refId = queryJobRefId;
     param->queryId = pJob->queryId;   
 
@@ -504,7 +504,7 @@ void* schtRunJobThread(void *aa) {
     }    
 
 
-    param = (SSchCallbackParam *)calloc(1, sizeof(*param));
+    param = (SSchTaskCallbackParam *)calloc(1, sizeof(*param));
     param->refId = queryJobRefId;
     param->queryId = pJob->queryId;   
     
@@ -524,7 +524,7 @@ void* schtRunJobThread(void *aa) {
     }  
 
 
-    param = (SSchCallbackParam *)calloc(1, sizeof(*param));
+    param = (SSchTaskCallbackParam *)calloc(1, sizeof(*param));
     param->refId = queryJobRefId;
     param->queryId = pJob->queryId;   
 
@@ -544,7 +544,7 @@ void* schtRunJobThread(void *aa) {
     }    
 
 
-    param = (SSchCallbackParam *)calloc(1, sizeof(*param));
+    param = (SSchTaskCallbackParam *)calloc(1, sizeof(*param));
     param->refId = queryJobRefId;
     param->queryId = pJob->queryId;   
 
@@ -713,6 +713,116 @@ TEST(queryTest, normalCase) {
   schedulerDestroy();
 }
 
+TEST(queryTest, readyFirstCase) {
+  void *mockPointer = (void *)0x1;
+  char *clusterId = "cluster1";
+  char *dbname = "1.db1";
+  char *tablename = "table1";
+  SVgroupInfo vgInfo = {0};
+  int64_t job = 0;
+  SQueryPlan dag;
+
+  memset(&dag, 0, sizeof(dag));
+
+  SArray *qnodeList = taosArrayInit(1, sizeof(SEp));
+
+  SEp qnodeAddr = {0};
+  strcpy(qnodeAddr.fqdn, "qnode0.ep");
+  qnodeAddr.port = 6031;
+  taosArrayPush(qnodeList, &qnodeAddr);
+  
+  int32_t code = schedulerInit(NULL);
+  ASSERT_EQ(code, 0);
+
+  schtBuildQueryDag(&dag);
+
+  schtSetPlanToString();
+  schtSetExecNode();
+  schtSetAsyncSendMsgToServer();
+  
+  code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, "select * from tb", &job);
+  ASSERT_EQ(code, 0);
+
+  
+  SSchJob *pJob = schAcquireJob(job);
+
+  void *pIter = taosHashIterate(pJob->execTasks, NULL);
+  while (pIter) {
+    SSchTask *task = *(SSchTask **)pIter;
+
+    SResReadyRsp rsp = {0};
+    code = schHandleResponseMsg(pJob, task, TDMT_VND_RES_READY_RSP, (char *)&rsp, sizeof(rsp), 0);
+    printf("code:%d", code);
+    ASSERT_EQ(code, 0);
+    pIter = taosHashIterate(pJob->execTasks, pIter);
+  }  
+  
+  pIter = taosHashIterate(pJob->execTasks, NULL);
+  while (pIter) {
+    SSchTask *task = *(SSchTask **)pIter;
+
+    SQueryTableRsp rsp = {0};
+    code = schHandleResponseMsg(pJob, task, TDMT_VND_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0);
+    
+    ASSERT_EQ(code, 0);
+    pIter = taosHashIterate(pJob->execTasks, pIter);
+  }    
+
+  pIter = taosHashIterate(pJob->execTasks, NULL);
+  while (pIter) {
+    SSchTask *task = *(SSchTask **)pIter;
+
+    SResReadyRsp rsp = {0};
+    code = schHandleResponseMsg(pJob, task, TDMT_VND_RES_READY_RSP, (char *)&rsp, sizeof(rsp), 0);
+    ASSERT_EQ(code, 0);
+    
+    pIter = taosHashIterate(pJob->execTasks, pIter);
+  }  
+
+  pIter = taosHashIterate(pJob->execTasks, NULL);
+  while (pIter) {
+    SSchTask *task = *(SSchTask **)pIter;
+
+    SQueryTableRsp rsp = {0};
+    code = schHandleResponseMsg(pJob, task, TDMT_VND_QUERY_RSP, (char *)&rsp, sizeof(rsp), 0);
+    
+    ASSERT_EQ(code, 0);
+    pIter = taosHashIterate(pJob->execTasks, pIter);
+  }    
+
+
+
+  TdThreadAttr thattr;
+  taosThreadAttrInit(&thattr);
+
+  TdThread thread1;
+  taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job);
+
+  void *data = NULL;  
+  code = schedulerFetchRows(job, &data);
+  ASSERT_EQ(code, 0);
+
+  SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data;
+  ASSERT_EQ(pRsp->completed, 1);
+  ASSERT_EQ(pRsp->numOfRows, 10);
+  tfree(data);
+
+  data = NULL;
+  code = schedulerFetchRows(job, &data);
+  ASSERT_EQ(code, 0);
+  ASSERT_TRUE(data == NULL);
+
+  schReleaseJob(job);
+
+  schedulerFreeJob(job);
+
+  schtFreeQueryDag(&dag);
+
+  schedulerDestroy();
+}
+
+
+
 TEST(queryTest, flowCtrlCase) {
   void *mockPointer = (void *)0x1;
   char *clusterId = "cluster1";

source/libs/sync/inc/syncRaftEntry.h

@@ -27,6 +27,12 @@ extern "C" {
 #include "syncMessage.h"
 #include "taosdef.h"
 
+typedef enum EntryType {
+  SYNC_RAFT_ENTRY_NOOP = 0,
+  SYNC_RAFT_ENTRY_DATA = 1,
+  SYNC_RAFT_ENTRY_CONFIG = 2,
+} EntryType;
+
 typedef struct SSyncRaftEntry {
   uint32_t  bytes;
   uint32_t  msgType;
@@ -35,12 +41,15 @@ typedef struct SSyncRaftEntry {
   bool      isWeak;
   SyncTerm  term;
   SyncIndex index;
+  EntryType entryType;
   uint32_t  dataLen;
   char      data[];
 } SSyncRaftEntry;
 
 SSyncRaftEntry* syncEntryBuild(uint32_t dataLen);
 SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index);  // step 4
+SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index, EntryType entryType);
+SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index);
 void            syncEntryDestory(SSyncRaftEntry* pEntry);
 char*           syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len);  // step 5
 SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len);              // step 6

source/libs/sync/src/syncAppendEntries.c

@@ -199,6 +199,9 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
             SSyncRaftEntry* pRollBackEntry = logStoreGetEntry(ths->pLogStore, index);
             assert(pRollBackEntry != NULL);
 
+            // maybe is a NOOP ENTRY
+            // assert(pRollBackEntry->entryType == SYNC_RAFT_ENTRY_DATA);
+
             SRpcMsg rpcMsg;
             syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg);
             ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, pRollBackEntry->index, pRollBackEntry->isWeak, 0, ths->state);
@@ -217,7 +220,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
         SRpcMsg rpcMsg;
         syncEntry2OriginalRpc(pAppendEntry, &rpcMsg);
         if (ths->pFsm != NULL) {
-          if (ths->pFsm->FpPreCommitCb != NULL) {
+          if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->entryType == SYNC_RAFT_ENTRY_DATA) {
             ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 2, ths->state);
           }
         }
@@ -242,7 +245,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
       SRpcMsg rpcMsg;
       syncEntry2OriginalRpc(pAppendEntry, &rpcMsg);
       if (ths->pFsm != NULL) {
-        if (ths->pFsm->FpPreCommitCb != NULL) {
+        if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->entryType == SYNC_RAFT_ENTRY_DATA) {
           ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 3, ths->state);
         }
       }
@@ -298,7 +301,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
               SRpcMsg rpcMsg;
               syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
-              if (ths->pFsm->FpCommitCb != NULL) {
+              if (ths->pFsm->FpCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) {
                 ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, ths->state);
               }
 

source/libs/sync/src/syncCommit.c

@@ -97,7 +97,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
           SRpcMsg rpcMsg;
           syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
-          if (pSyncNode->pFsm->FpCommitCb != NULL) {
+          if (pSyncNode->pFsm->FpCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) {
             pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, pSyncNode->state);
           }
 

source/libs/sync/src/syncIO.c

@@ -29,7 +29,7 @@ static int32_t  syncIODestroy(SSyncIO *io);
 static int32_t  syncIOStartInternal(SSyncIO *io);
 static int32_t  syncIOStopInternal(SSyncIO *io);
 
-static void *  syncIOConsumerFunc(void *param);
+static void   *syncIOConsumerFunc(void *param);
 static void    syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet);
 static void    syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet);
 static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey);
@@ -75,6 +75,7 @@ int32_t syncIOSendMsg(void *clientRpc, const SEpSet *pEpSet, SRpcMsg *pMsg) {
   syncRpcMsgPrint2(logBuf, pMsg);
 
   pMsg->handle = NULL;
+  pMsg->noResp = 1;
   rpcSendRequest(clientRpc, pEpSet, pMsg, NULL);
   return ret;
 }
@@ -234,9 +235,9 @@ static int32_t syncIOStopInternal(SSyncIO *io) {
 }
 
 static void *syncIOConsumerFunc(void *param) {
-  SSyncIO *  io = param;
+  SSyncIO   *io = param;
   STaosQall *qall;
-  SRpcMsg *  pRpcMsg, rpcMsg;
+  SRpcMsg   *pRpcMsg, rpcMsg;
   qall = taosAllocateQall();
 
   while (1) {
@@ -324,19 +325,21 @@ static void *syncIOConsumerFunc(void *param) {
       taosGetQitem(qall, (void **)&pRpcMsg);
       rpcFreeCont(pRpcMsg->pCont);
 
-      if (pRpcMsg->handle != NULL) {
-        int msgSize = 32;
-        memset(&rpcMsg, 0, sizeof(rpcMsg));
-        rpcMsg.msgType = SYNC_RESPONSE;
-        rpcMsg.pCont = rpcMallocCont(msgSize);
-        rpcMsg.contLen = msgSize;
-        snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", "give a reply");
-        rpcMsg.handle = pRpcMsg->handle;
-        rpcMsg.code = 0;
-
-        syncRpcMsgPrint2((char *)"syncIOConsumerFunc rpcSendResponse --> ", &rpcMsg);
-        rpcSendResponse(&rpcMsg);
-      }
+      /*
+            if (pRpcMsg->handle != NULL) {
+              int msgSize = 32;
+              memset(&rpcMsg, 0, sizeof(rpcMsg));
+              rpcMsg.msgType = SYNC_RESPONSE;
+              rpcMsg.pCont = rpcMallocCont(msgSize);
+              rpcMsg.contLen = msgSize;
+              snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", "give a reply");
+              rpcMsg.handle = pRpcMsg->handle;
+              rpcMsg.code = 0;
+
+              syncRpcMsgPrint2((char *)"syncIOConsumerFunc rpcSendResponse --> ", &rpcMsg);
+              rpcSendResponse(&rpcMsg);
+            }
+      */
 
       taosFreeQitem(pRpcMsg);
     }

source/libs/sync/src/syncMain.c

@@ -37,11 +37,13 @@ static int32_t tsNodeRefId = -1;
 
 // ------ local funciton ---------
 // enqueue message ----
-static void syncNodeEqPingTimer(void* param, void* tmrId);
-static void syncNodeEqElectTimer(void* param, void* tmrId);
-static void syncNodeEqHeartbeatTimer(void* param, void* tmrId);
+static void    syncNodeEqPingTimer(void* param, void* tmrId);
+static void    syncNodeEqElectTimer(void* param, void* tmrId);
+static void    syncNodeEqHeartbeatTimer(void* param, void* tmrId);
+static int32_t syncNodeEqNoop(SSyncNode* ths);
+static int32_t syncNodeAppendNoop(SSyncNode* ths);
 
-// on message ----
+// process message ----
 static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg);
 static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg);
 static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg);
@@ -669,6 +671,10 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) {
   assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE);
   assert(voteGrantedMajority(pSyncNode->pVotesGranted));
   syncNodeBecomeLeader(pSyncNode);
+
+  // Raft 3.6.2 Committing entries from previous terms
+  syncNodeAppendNoop(pSyncNode);
+  // syncNodeEqNoop(pSyncNode);
 }
 
 void syncNodeFollower2Candidate(SSyncNode* pSyncNode) {
@@ -803,6 +809,47 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) {
   }
 }
 
+static int32_t syncNodeEqNoop(SSyncNode* ths) {
+  int32_t ret = 0;
+  assert(ths->state == TAOS_SYNC_STATE_LEADER);
+
+  SyncIndex       index = ths->pLogStore->getLastIndex(ths->pLogStore) + 1;
+  SyncTerm        term = ths->pRaftStore->currentTerm;
+  SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index);
+  assert(pEntry != NULL);
+
+  uint32_t           entryLen;
+  char*              serialized = syncEntrySerialize(pEntry, &entryLen);
+  SyncClientRequest* pSyncMsg = syncClientRequestBuild(entryLen);
+  assert(pSyncMsg->dataLen == entryLen);
+  memcpy(pSyncMsg->data, serialized, entryLen);
+
+  SRpcMsg rpcMsg;
+  syncClientRequest2RpcMsg(pSyncMsg, &rpcMsg);
+  ths->FpEqMsg(ths->queue, &rpcMsg);
+
+  free(serialized);
+  syncClientRequestDestroy(pSyncMsg);
+
+  return ret;
+}
+
+static int32_t syncNodeAppendNoop(SSyncNode* ths) {
+  int32_t ret = 0;
+
+  SyncIndex       index = ths->pLogStore->getLastIndex(ths->pLogStore) + 1;
+  SyncTerm        term = ths->pRaftStore->currentTerm;
+  SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index);
+  assert(pEntry != NULL);
+
+  if (ths->state == TAOS_SYNC_STATE_LEADER) {
+    ths->pLogStore->appendEntry(ths->pLogStore, pEntry);
+    syncNodeReplicate(ths);
+  }
+
+  return ret;
+}
+
 // on message ----
 static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) {
   int32_t ret = 0;
@@ -851,7 +898,7 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg
     syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
     if (ths->pFsm != NULL) {
-      if (ths->pFsm->FpPreCommitCb != NULL) {
+      if (ths->pFsm->FpPreCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) {
         ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, ths->state);
       }
     }
@@ -866,7 +913,7 @@ static int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg
     syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
     if (ths->pFsm != NULL) {
-      if (ths->pFsm->FpPreCommitCb != NULL) {
+      if (ths->pFsm->FpPreCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) {
         ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 1, ths->state);
       }
     }

tests/script/tsim/insert/backquote.sim

@@ -174,10 +174,6 @@ while $dbCnt < 2
     #if $rows != 4 then 
     #  return -1
     #endi
-    ##sql select * from st
-    ##if $rows != 4 then
-    ##  return -1
-    ##endi
 
 endw