Merge branch '3.0' into test/jcy

Jenkinsfile2

@@ -88,12 +88,6 @@ def pre_test(){
     cmake .. > /dev/null
     make -j4> /dev/null
     '''
-    sh'''
-    cd ${WKPY}
-    git reset --hard
-    git pull
-    pip3 install .
-    '''
     return 1
 }
 
@@ -103,7 +97,6 @@ pipeline {
   environment{
       WK = '/var/lib/jenkins/workspace/TDinternal'
       WKC= '/var/lib/jenkins/workspace/TDengine'
-      WKPY= '/var/lib/jenkins/workspace/taos-connector-python'
   }
   stages {
       stage('pre_build'){
@@ -124,11 +117,6 @@ pipeline {
               ./test-all.sh b1fq
               '''
               sh'''
-              export LD_LIBRARY_PATH=${WKC}/debug/build/lib
-              cd ${WKC}/tests/system-test
-              ./fulltest.sh
-              '''
-              sh'''
               cd ${WKC}/debug
               ctest
               '''

include/client/taos.h

@@ -146,6 +146,7 @@ DLL_EXPORT TAOS_RES   *taos_stmt_use_result(TAOS_STMT *stmt);
 DLL_EXPORT int         taos_stmt_close(TAOS_STMT *stmt);
 DLL_EXPORT char       *taos_stmt_errstr(TAOS_STMT *stmt);
 DLL_EXPORT int         taos_stmt_affected_rows(TAOS_STMT *stmt);
+DLL_EXPORT int         taos_stmt_affected_rows_once(TAOS_STMT *stmt);
 
 DLL_EXPORT TAOS_RES   *taos_query(TAOS *taos, const char *sql);
 DLL_EXPORT TAOS_RES   *taos_query_l(TAOS *taos, const char *sql, int sqlLen);

include/common/taosdef.h

@@ -78,13 +78,6 @@ typedef enum {
   TSDB_SMA_TYPE_ROLLUP = 2,      // Rollup SMA
 } ETsdbSmaType;
 
-typedef enum {
-  TSDB_BSMA_TYPE_NONE = 0,  // no block-wise SMA
-  TSDB_BSMA_TYPE_I = 1,     // sum/min/max(default)
-} ETsdbBSmaType;
-
-#define TSDB_BSMA_TYPE_LATEST TSDB_BSMA_TYPE_I
-
 extern char *qtypeStr[];
 
 #define TSDB_PORT_HTTP      11

include/common/tmsg.h

@@ -326,6 +326,13 @@ int32_t tDecodeSEpSet(SCoder* pDecoder, SEpSet* pEp);
 int32_t taosEncodeSEpSet(void** buf, const SEpSet* pEp);
 void*   taosDecodeSEpSet(const void* buf, SEpSet* pEp);
 
+typedef struct {
+  SEpSet epSet;
+} SMEpSet;
+
+int32_t tSerializeSMEpSet(void* buf, int32_t bufLen, SMEpSet* pReq);
+int32_t tDeserializeSMEpSet(void* buf, int32_t buflen, SMEpSet* pReq);
+
 typedef struct {
   int8_t  connType;
   int32_t pid;
@@ -682,6 +689,7 @@ int32_t tDeserializeSDropFuncReq(void* buf, int32_t bufLen, SDropFuncReq* pReq);
 
 typedef struct {
   int32_t numOfFuncs;
+  bool    ignoreCodeComment;
   SArray* pFuncNames;
 } SRetrieveFuncReq;
 
@@ -710,6 +718,7 @@ typedef struct {
 
 int32_t tSerializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp* pRsp);
 int32_t tDeserializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp* pRsp);
+void tFreeSFuncInfo(SFuncInfo *pInfo);
 void    tFreeSRetrieveFuncRsp(SRetrieveFuncRsp* pRsp);
 
 typedef struct {
@@ -982,7 +991,6 @@ int32_t tDeserializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp);
 void    tFreeSShowRsp(SShowRsp* pRsp);
 
 typedef struct {
-  int32_t type;
   char    db[TSDB_DB_FNAME_LEN];
   char    tb[TSDB_TABLE_NAME_LEN];
   int64_t showId;
@@ -1478,8 +1486,12 @@ typedef struct {
 typedef struct {
   float      xFilesFactor;
   int32_t    delay;
-  int8_t     nFuncIds;
+  int32_t    qmsg1Len;
+  int32_t    qmsg2Len;
   func_id_t* pFuncIds;
+  char*      qmsg1;  // not null: pAst1:qmsg1:SRetention1 => trigger aggr task1
+  char*      qmsg2;  // not null: pAst2:qmsg2:SRetention2 => trigger aggr task2
+  int8_t     nFuncIds;
 } SRSmaParam;
 
 typedef struct SVCreateTbReq {
@@ -2329,9 +2341,10 @@ static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW) {
   }
 }
 
-static FORCE_INLINE void tdFreeTSmaWrapper(STSmaWrapper* pSW) {
+static FORCE_INLINE void* tdFreeTSmaWrapper(STSmaWrapper* pSW) {
   tdDestroyTSmaWrapper(pSW);
-  taosMemoryFreeClear(pSW);
+  taosMemoryFree(pSW);
+  return NULL;
 }
 
 static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) {
@@ -2719,6 +2732,7 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p
   }
   return buf;
 }
+
 #pragma pack(pop)
 
 #ifdef __cplusplus

include/libs/catalog/catalog.h

@@ -224,6 +224,8 @@ int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
 
 int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo);
 
+int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo);
+
 
 /**
  * Destroy catalog and relase all resources

include/libs/function/function.h

@@ -166,6 +166,7 @@ typedef struct SInputColumnInfoData {
   SColumnInfoData  *pPTS;           // primary timestamp column
   SColumnInfoData **pData;
   SColumnDataAgg  **pColumnDataAgg;
+  uint64_t          uid;            // table uid
 } SInputColumnInfoData;
 
 // sql function runtime context
@@ -191,7 +192,7 @@ typedef struct SqlFunctionCtx {
   int16_t          functionId;    // function id
   char *           pOutput;       // final result output buffer, point to sdata->data
   int32_t          numOfParams;
-  SVariant         param[4];      // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
+  SFunctParam     *param;         // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
   int64_t         *ptsList;       // corresponding timestamp array list
   SColumnInfoData *pTsOutput;     // corresponding output buffer for timestamp of each result, e.g., top/bottom*/
   int32_t          offset;

include/libs/function/functionMgt.h

@@ -119,12 +119,19 @@ typedef enum EFunctionType {
 struct SqlFunctionCtx;
 struct SResultRowEntryInfo;
 struct STimeWindow;
+struct SCatalog;
+
+typedef struct SFmGetFuncInfoParam {
+  struct SCatalog* pCtg;
+  void *pRpc;
+  const SEpSet* pMgmtEps;
+} SFmGetFuncInfoParam;
 
 int32_t fmFuncMgtInit();
 
 void fmFuncMgtDestroy();
 
-int32_t fmGetFuncInfo(const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType);
+int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType);
 
 int32_t fmGetFuncResultType(SFunctionNode* pFunc, char* pErrBuf, int32_t len);
 

include/libs/transport/trpc.h

@@ -54,12 +54,13 @@ typedef struct {
   uint16_t clientPort;
   SRpcMsg  rpcMsg;
   int32_t  rspLen;
-  void    *pRsp;
-  void    *pNode;
+  void *   pRsp;
+  void *   pNode;
 } SNodeMsg;
 
 typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *);
 typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
+typedef int (*RpcRfp)(void *parent, SRpcMsg *, SEpSet *);
 
 typedef struct SRpcInit {
   uint16_t localPort;     // local port
@@ -80,22 +81,25 @@ typedef struct SRpcInit {
   RpcCfp cfp;
 
   // call back to retrieve the client auth info, for server app only
-  RpcAfp afp;;
+  RpcAfp afp;
+
+  // user defined retry func
+  RpcRfp rfp;
 
   void *parent;
 } SRpcInit;
 
 typedef struct {
-  void     *val;
+  void *val;
   int32_t (*clone)(void *src, void **dst);
-  void    (*freeFunc)(const void *arg);
+  void (*freeFunc)(const void *arg);
 } SRpcCtxVal;
 
 typedef struct {
-  int32_t   msgType;
-  void     *val;
+  int32_t msgType;
+  void *  val;
   int32_t (*clone)(void *src, void **dst);
-  void    (*freeFunc)(const void *arg);
+  void (*freeFunc)(const void *arg);
 } SRpcBrokenlinkVal;
 
 typedef struct {

include/os/osEnv.h

@@ -34,6 +34,7 @@ extern int64_t         tsOpenMax;
 extern int64_t         tsStreamMax;
 extern float           tsNumOfCores;
 extern int64_t         tsTotalMemoryKB;
+extern char*           tsProcPath;
 
 extern char configDir[];
 extern char tsDataDir[];

include/util/talgo.h

@@ -82,7 +82,7 @@ void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size
  * @return
  */
 void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar,
-                    __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot);
+                    __ext_compar_fn_t compar, char* buf, bool maxroot);
 
 /**
  * sort heap to make sure it is a max/min root heap
@@ -98,7 +98,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const
  * @return
  */
 void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar,
-                  const void *parswap, __ext_swap_fn_t swap, bool maxroot);
+                  bool maxroot);
 
 #ifdef __cplusplus
 }

source/client/inc/clientInt.h

@@ -219,6 +219,7 @@ void    doSetOneRowPtr(SReqResultInfo* pResultInfo);
 void    setResPrecision(SReqResultInfo* pResInfo, int32_t precision);
 int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4);
 void    setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols);
+void    doFreeReqResultInfo(SReqResultInfo* pResInfo);
 
 static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) {
   SMqRspObj* msg = (SMqRspObj*)res;

source/client/inc/clientStmt.h

@@ -60,6 +60,7 @@ typedef struct SStmtBindInfo {
 } SStmtBindInfo;
 
 typedef struct SStmtExecInfo {
+  int32_t      affectedRows;
   SRequestObj* pRequest;
   SHashObj*    pVgHash;
   SHashObj*    pBlockHash;

source/client/src/clientEnv.c

@@ -30,15 +30,15 @@
 #define TSC_VAR_RELEASED    0
 
 SAppInfo appInfo;
-int32_t  clientReqRefPool  = -1;
+int32_t  clientReqRefPool = -1;
 int32_t  clientConnRefPool = -1;
 
-static TdThreadOnce   tscinit = PTHREAD_ONCE_INIT;
-volatile int32_t      tscInitRes = 0;
+static TdThreadOnce tscinit = PTHREAD_ONCE_INIT;
+volatile int32_t    tscInitRes = 0;
 
 static void registerRequest(SRequestObj *pRequest) {
   STscObj *pTscObj = acquireTscObj(pRequest->pTscObj->id);
-  
+
   assert(pTscObj != NULL);
 
   // connection has been released already, abort creating request.
@@ -49,8 +49,8 @@ static void registerRequest(SRequestObj *pRequest) {
   if (pTscObj->pAppInfo) {
     SInstanceSummary *pSummary = &pTscObj->pAppInfo->summary;
 
-    int32_t total = atomic_add_fetch_64((int64_t*)&pSummary->totalRequests, 1);
-    int32_t currentInst = atomic_add_fetch_64((int64_t*)&pSummary->currentRequests, 1);
+    int32_t total = atomic_add_fetch_64((int64_t *)&pSummary->totalRequests, 1);
+    int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1);
     tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64
              ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64,
              pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
@@ -60,16 +60,16 @@ static void registerRequest(SRequestObj *pRequest) {
 static void deregisterRequest(SRequestObj *pRequest) {
   assert(pRequest != NULL);
 
-  STscObj *         pTscObj = pRequest->pTscObj;
+  STscObj          *pTscObj = pRequest->pTscObj;
   SInstanceSummary *pActivity = &pTscObj->pAppInfo->summary;
 
-  int32_t currentInst = atomic_sub_fetch_64((int64_t*)&pActivity->currentRequests, 1);
+  int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1);
   int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1);
 
   int64_t duration = taosGetTimestampUs() - pRequest->metric.start;
   tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64
            " ms, current:%d, app current:%d",
-           pRequest->self, pTscObj->id, pRequest->requestId, duration/1000, num, currentInst);
+           pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst);
   releaseTscObj(pTscObj->id);
 }
 
@@ -109,12 +109,12 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) {
 }
 
 void closeAllRequests(SHashObj *pRequests) {
-  void   *pIter = taosHashIterate(pRequests, NULL);
+  void *pIter = taosHashIterate(pRequests, NULL);
   while (pIter != NULL) {
     int64_t *rid = pIter;
 
-    releaseRequest(*rid);    
-    
+    releaseRequest(*rid);
+
     pIter = taosHashIterate(pRequests, pIter);
   }
 }
@@ -144,7 +144,7 @@ void *createTscObj(const char *user, const char *auth, const char *db, SAppInstI
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return NULL;
   }
-  
+
   pObj->pAppInfo = pAppInfo;
   tstrncpy(pObj->user, user, sizeof(pObj->user));
   memcpy(pObj->pass, auth, TSDB_PASSWORD_LEN);
@@ -160,13 +160,9 @@ void *createTscObj(const char *user, const char *auth, const char *db, SAppInstI
   return pObj;
 }
 
-STscObj *acquireTscObj(int64_t rid) {
-  return (STscObj *)taosAcquireRef(clientConnRefPool, rid);
-}
+STscObj *acquireTscObj(int64_t rid) { return (STscObj *)taosAcquireRef(clientConnRefPool, rid); }
 
-int32_t releaseTscObj(int64_t rid) {
-  return taosReleaseRef(clientConnRefPool, rid);
-}
+int32_t releaseTscObj(int64_t rid) { return taosReleaseRef(clientConnRefPool, rid); }
 
 void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t type) {
   assert(pObj != NULL);
@@ -190,11 +186,11 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty
   tsem_init(&pRequest->body.rspSem, 0, 0);
 
   registerRequest(pRequest);
-    
+
   return pRequest;
 }
 
-static void doFreeReqResultInfo(SReqResultInfo *pResInfo) {
+void doFreeReqResultInfo(SReqResultInfo *pResInfo) {
   taosMemoryFreeClear(pResInfo->pRspMsg);
   taosMemoryFreeClear(pResInfo->length);
   taosMemoryFreeClear(pResInfo->row);
@@ -217,7 +213,7 @@ static void doDestroyRequest(void *p) {
   assert(RID_VALID(pRequest->self));
 
   taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self));
-  
+
   taosMemoryFreeClear(pRequest->msgBuf);
   taosMemoryFreeClear(pRequest->sqlstr);
   taosMemoryFreeClear(pRequest->pDb);
@@ -244,14 +240,9 @@ void destroyRequest(SRequestObj *pRequest) {
   taosRemoveRef(clientReqRefPool, pRequest->self);
 }
 
-SRequestObj *acquireRequest(int64_t rid) {
-  return (SRequestObj *)taosAcquireRef(clientReqRefPool, rid);
-}
-
-int32_t releaseRequest(int64_t rid) {
-  return taosReleaseRef(clientReqRefPool, rid);
-}
+SRequestObj *acquireRequest(int64_t rid) { return (SRequestObj *)taosAcquireRef(clientReqRefPool, rid); }
 
+int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, rid); }
 
 void taos_init_imp(void) {
   // In the APIs of other program language, taos_cleanup is not available yet.
@@ -380,7 +371,7 @@ uint64_t generateRequestId() {
   }
 
   uint64_t id = 0;
-  
+
   while (true) {
     int64_t  ts = taosGetTimestampMs();
     uint64_t pid = taosGetPId();

source/client/src/clientMain.c

@@ -135,6 +135,16 @@ void taos_free_result(TAOS_RES *res) {
   if (TD_RES_QUERY(res)) {
     SRequestObj *pRequest = (SRequestObj *)res;
     destroyRequest(pRequest);
+  } else if (TD_RES_TMQ(res)) {
+    SMqRspObj *pRsp = (SMqRspObj *)res;
+    if (pRsp->rsp.blockData) taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree);
+    if (pRsp->rsp.blockDataLen) taosArrayDestroy(pRsp->rsp.blockDataLen);
+    if (pRsp->rsp.blockSchema) taosArrayDestroy(pRsp->rsp.blockSchema);
+    if (pRsp->rsp.blockTbName) taosArrayDestroy(pRsp->rsp.blockTbName);
+    if (pRsp->rsp.blockTags) taosArrayDestroy(pRsp->rsp.blockTags);
+    if (pRsp->rsp.blockTagSchema) taosArrayDestroy(pRsp->rsp.blockTagSchema);
+    pRsp->resInfo.pRspMsg = NULL;
+    doFreeReqResultInfo(&pRsp->resInfo);
   }
 }
 
@@ -622,6 +632,10 @@ int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) {
   return stmtSetTbName(stmt, name);
 }
 
+int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name) {
+  return taos_stmt_set_tbname(stmt, name);
+}
+
 int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) {
   if (stmt == NULL || bind == NULL) {
     tscError("NULL parameter for %s", __FUNCTION__);

source/client/src/clientStmt.c

@@ -486,7 +486,8 @@ int stmtExec(TAOS_STMT *stmt) {
 
   STMT_ERR_JRET(pStmt->exec.pRequest->code);
 
-  pStmt->affectedRows += taos_affected_rows(pStmt->exec.pRequest);
+  pStmt->exec.affectedRows = taos_affected_rows(pStmt->exec.pRequest);
+  pStmt->affectedRows += pStmt->exec.affectedRows;
 
 _return:
 

source/client/test/clientTests.cpp

@@ -662,7 +662,7 @@ TEST(testCase, agg_query_tables) {
   TAOS_RES* pRes = taos_query(pConn, "use abc1");
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "select count(*) from tu");
+  pRes = taos_query(pConn, "select now() from m1");
   if (taos_errno(pRes) != 0) {
     printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
     taos_free_result(pRes);

source/common/src/tdataformat.c

@@ -110,7 +110,7 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) {
 
   for (int i = 0; i < numOfCols; i++) {
     col_type_t  type = 0;
-    int8_t      sma = TSDB_BSMA_TYPE_NONE;
+    int8_t      sma = 0;
     col_id_t    colId = 0;
     col_bytes_t bytes = 0;
     buf = taosDecodeFixedI8(buf, &type);

source/common/src/tmsg.c

@@ -434,6 +434,15 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
         for (int8_t i = 0; i < param->nFuncIds; ++i) {
           tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]);
         }
+        tlen += taosEncodeFixedI32(buf, param->qmsg1Len);
+        if (param->qmsg1Len > 0) {
+          tlen += taosEncodeString(buf, param->qmsg1);
+        }
+
+        tlen += taosEncodeFixedI32(buf, param->qmsg2Len);
+        if (param->qmsg2Len > 0) {
+          tlen += taosEncodeString(buf, param->qmsg2);
+        }
       }
       break;
     case TD_CHILD_TABLE:
@@ -496,18 +505,25 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
         buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name);
       }
       if (pReq->rollup) {
-        pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam));
+        pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryCalloc(1, sizeof(SRSmaParam));
         SRSmaParam *param = pReq->stbCfg.pRSmaParam;
         buf = taosDecodeBinaryTo(buf, (void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
         buf = taosDecodeFixedI32(buf, &param->delay);
         buf = taosDecodeFixedI8(buf, &param->nFuncIds);
         if (param->nFuncIds > 0) {
-          param->pFuncIds = (func_id_t *)taosMemoryMalloc(param->nFuncIds * sizeof(func_id_t));
+          param->pFuncIds = (func_id_t *)taosMemoryCalloc(param->nFuncIds, sizeof(func_id_t));
           for (int8_t i = 0; i < param->nFuncIds; ++i) {
             buf = taosDecodeFixedI32(buf, param->pFuncIds + i);
           }
-        } else {
-          param->pFuncIds = NULL;
+        }
+        buf = taosDecodeFixedI32(buf, &param->qmsg1Len);
+        if (param->qmsg1Len > 0) {
+          buf = taosDecodeString(buf, &param->qmsg1);
+        }
+
+        buf = taosDecodeFixedI32(buf, &param->qmsg2Len);
+        if (param->qmsg2Len > 0) {
+          buf = taosDecodeString(buf, &param->qmsg2);
         }
       } else {
         pReq->stbCfg.pRSmaParam = NULL;
@@ -828,6 +844,27 @@ void tFreeSMAltertbReq(SMAltertbReq *pReq) {
   taosArrayDestroy(pReq->pFields);
   pReq->pFields = NULL;
 }
+int32_t tSerializeSMEpSet(void *buf, int32_t bufLen, SMEpSet *pReq) {
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeSEpSet(&encoder, &pReq->epSet) < 0) return -1;
+
+  tEndEncode(&encoder);
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
+}
+int32_t tDeserializeSMEpSet(void *buf, int32_t bufLen, SMEpSet *pReq) {
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeSEpSet(&decoder, &pReq->epSet) < 0) return -1;
+
+  tEndDecode(&decoder);
+  tCoderClear(&decoder);
+  return 0;
+}
 
 int32_t tSerializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pReq) {
   SCoder encoder = {0};
@@ -1624,6 +1661,7 @@ int32_t tSerializeSRetrieveFuncReq(void *buf, int32_t bufLen, SRetrieveFuncReq *
 
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->numOfFuncs) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->ignoreCodeComment) < 0) return -1;
 
   if (pReq->numOfFuncs != (int32_t)taosArrayGetSize(pReq->pFuncNames)) return -1;
   for (int32_t i = 0; i < pReq->numOfFuncs; ++i) {
@@ -1644,6 +1682,7 @@ int32_t tDeserializeSRetrieveFuncReq(void *buf, int32_t bufLen, SRetrieveFuncReq
 
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->numOfFuncs) < 0) return -1;
+  if (tDecodeI8(&decoder, (int8_t *)&pReq->ignoreCodeComment) < 0) return -1;
 
   pReq->pFuncNames = taosArrayInit(pReq->numOfFuncs, TSDB_FUNC_NAME_LEN);
   if (pReq->pFuncNames == NULL) return -1;
@@ -1681,8 +1720,12 @@ int32_t tSerializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp *
     if (tEncodeI64(&encoder, pInfo->signature) < 0) return -1;
     if (tEncodeI32(&encoder, pInfo->codeSize) < 0) return -1;
     if (tEncodeI32(&encoder, pInfo->commentSize) < 0) return -1;
-    if (tEncodeCStr(&encoder, pInfo->pCode) < 0) return -1;
-    if (tEncodeCStr(&encoder, pInfo->pComment) < 0) return -1;
+    if (pInfo->codeSize) {
+      if (tEncodeCStr(&encoder, pInfo->pCode) < 0) return -1;
+    }
+    if (pInfo->commentSize) {
+      if (tEncodeCStr(&encoder, pInfo->pComment) < 0) return -1;
+    }
   }
 
   tEndEncode(&encoder);
@@ -1713,15 +1756,23 @@ int32_t tDeserializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp
     if (tDecodeI64(&decoder, &fInfo.signature) < 0) return -1;
     if (tDecodeI32(&decoder, &fInfo.codeSize) < 0) return -1;
     if (tDecodeI32(&decoder, &fInfo.commentSize) < 0) return -1;
-    fInfo.pCode = taosMemoryCalloc(1, fInfo.codeSize);
-    fInfo.pComment = taosMemoryCalloc(1, fInfo.commentSize);
-    if (fInfo.pCode == NULL || fInfo.pComment == NULL) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      return -1;
+    if (fInfo.codeSize) {
+      fInfo.pCode = taosMemoryCalloc(1, fInfo.codeSize);
+      if (fInfo.pCode == NULL) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        return -1;
+      }
+      if (tDecodeCStrTo(&decoder, fInfo.pCode) < 0) return -1;
+    }
+    if (fInfo.commentSize) {
+      fInfo.pComment = taosMemoryCalloc(1, fInfo.commentSize);
+      if (fInfo.pComment == NULL) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        return -1;
+      }
+      if (tDecodeCStrTo(&decoder, fInfo.pComment) < 0) return -1;
     }
 
-    if (tDecodeCStrTo(&decoder, fInfo.pCode) < 0) return -1;
-    if (tDecodeCStrTo(&decoder, fInfo.pComment) < 0) return -1;
     taosArrayPush(pRsp->pFuncInfos, &fInfo);
   }
   tEndDecode(&decoder);
@@ -1730,12 +1781,20 @@ int32_t tDeserializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp
   return 0;
 }
 
+void tFreeSFuncInfo(SFuncInfo *pInfo) {
+  if (NULL == pInfo) {
+    return;
+  }
+
+  taosMemoryFree(pInfo->pCode);
+  taosMemoryFree(pInfo->pComment);
+}
+
 void tFreeSRetrieveFuncRsp(SRetrieveFuncRsp *pRsp) {
   int32_t size = taosArrayGetSize(pRsp->pFuncInfos);
   for (int32_t i = 0; i < size; ++i) {
     SFuncInfo *pInfo = taosArrayGet(pRsp->pFuncInfos, i);
-    taosMemoryFree(pInfo->pCode);
-    taosMemoryFree(pInfo->pComment);
+    tFreeSFuncInfo(pInfo);
   }
   taosArrayDestroy(pRsp->pFuncInfos);
 }
@@ -2403,7 +2462,6 @@ int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq
 
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->showId) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->type) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->tb) < 0) return -1;
   tEndEncode(&encoder);
@@ -2419,7 +2477,6 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR
 
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeI64(&decoder, &pReq->showId) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->type) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->tb) < 0) return -1;
   tEndDecode(&decoder);

source/dnode/mgmt/implement/src/dmHandle.c

@@ -216,6 +216,126 @@ static void dmStopMgmt(SMgmtWrapper *pWrapper) {
   dmStopStatusThread(pWrapper->pDnode);
 }
 
+static int32_t dmSpawnUdfd(SDnode *pDnode);
+
+void dmUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal) {
+  dInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal);
+  uv_close((uv_handle_t*)process, NULL);
+  SDnode *pDnode = process->data;
+  SUdfdData *pData = &pDnode->udfdData;
+  if (atomic_load_8(&pData->stopping) != 0) {
+    dDebug("udfd process exit due to stopping");
+  } else {
+    uv_close((uv_handle_t*)&pData->ctrlPipe, NULL);
+    dmSpawnUdfd(pDnode);
+  }
+}
+
+static int32_t dmSpawnUdfd(SDnode *pDnode) {
+  dInfo("dnode start spawning udfd");
+  uv_process_options_t options = {0};
+
+  char path[PATH_MAX] = {0};
+  if (tsProcPath == NULL) {
+    path[0] = '.';
+  } else {
+    strncpy(path, tsProcPath, strlen(tsProcPath));
+    taosDirName(path);
+  }
+  strcat(path, "/udfd");
+  char* argsUdfd[] = {path, "-c", configDir, NULL};
+  options.args = argsUdfd;
+  options.file = path;
+
+  options.exit_cb = dmUdfdExit;
+  SUdfdData *pData = &pDnode->udfdData;
+  uv_pipe_init(&pData->loop, &pData->ctrlPipe, 1);
+
+  uv_stdio_container_t child_stdio[3];
+  child_stdio[0].flags = UV_CREATE_PIPE | UV_READABLE_PIPE;
+  child_stdio[0].data.stream = (uv_stream_t*) &pData->ctrlPipe;
+  child_stdio[1].flags = UV_IGNORE;
+  child_stdio[2].flags = UV_INHERIT_FD;
+  child_stdio[2].data.fd = 2;
+  options.stdio_count = 3;
+  options.stdio = child_stdio;
+
+  char dnodeIdEnvItem[32] = {0};
+  char thrdPoolSizeEnvItem[32] = {0};
+  snprintf(dnodeIdEnvItem, 32, "%s=%d", "DNODE_ID", pDnode->data.dnodeId);
+  float numCpuCores = 4;
+  taosGetCpuCores(&numCpuCores);
+  snprintf(thrdPoolSizeEnvItem,32,  "%s=%d", "UV_THREADPOOL_SIZE", (int)numCpuCores*2);
+  char* envUdfd[] = {dnodeIdEnvItem, thrdPoolSizeEnvItem, NULL};
+  options.env = envUdfd;
+
+  int err = uv_spawn(&pData->loop, &pData->process, &options);
+  pData->process.data = (void*)pDnode;
+
+  if (err != 0) {
+    dError("can not spawn udfd. path: %s, error: %s", path, uv_strerror(err));
+  }
+  return err;
+}
+
+static void dmUdfdCloseWalkCb(uv_handle_t* handle, void* arg) {
+  if (!uv_is_closing(handle)) {
+    uv_close(handle, NULL);
+  }
+}
+
+void dmWatchUdfd(void *args) {
+  SDnode *pDnode = args;
+  SUdfdData *pData = &pDnode->udfdData;
+  uv_loop_init(&pData->loop);
+  int32_t err = dmSpawnUdfd(pDnode);
+  atomic_store_32(&pData->spawnErr, err);
+  uv_barrier_wait(&pData->barrier);
+  uv_run(&pData->loop, UV_RUN_DEFAULT);
+  err = uv_loop_close(&pData->loop);
+  while (err == UV_EBUSY) {
+    uv_walk(&pData->loop, dmUdfdCloseWalkCb, NULL);
+    uv_run(&pData->loop, UV_RUN_DEFAULT);
+    err = uv_loop_close(&pData->loop);
+  }
+  return;
+}
+
+int32_t dmStartUdfd(SDnode *pDnode) {
+  SUdfdData *pData = &pDnode->udfdData;
+  if (pData->startCalled) {
+    dInfo("dnode-mgmt start udfd already called");
+    return 0;
+  }
+  uv_barrier_init(&pData->barrier, 2);
+  pData->stopping = 0;
+  uv_thread_create(&pData->thread, dmWatchUdfd, pDnode);
+  uv_barrier_wait(&pData->barrier);
+  pData->startCalled = true;
+  pData->needCleanUp = true;
+  return pData->spawnErr;
+}
+
+int32_t dmStopUdfd(SDnode *pDnode) {
+  dInfo("dnode-mgmt to stop udfd. need cleanup: %d, spawn err: %d",
+        pDnode->udfdData.needCleanUp, pDnode->udfdData.spawnErr);
+  SUdfdData *pData = &pDnode->udfdData;
+  if (!pData->needCleanUp) {
+    return 0;
+  }
+  atomic_store_8(&pData->stopping, 1);
+
+  uv_barrier_destroy(&pData->barrier);
+  if (pData->spawnErr == 0) {
+    uv_process_kill(&pData->process, SIGINT);
+  }
+  uv_stop(&pData->loop);
+  uv_thread_join(&pData->thread);
+
+  atomic_store_8(&pData->stopping, 0);
+  return 0;
+}
+
 static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) {
   dInfo("dnode-mgmt start to init");
   SDnode *pDnode = pWrapper->pDnode;
@@ -247,6 +367,10 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) {
   }
   dmReportStartup(pDnode, "dnode-transport", "initialized");
 
+  if (dmStartUdfd(pDnode) != 0) {
+    dError("failed to start udfd");
+  }
+
   dInfo("dnode-mgmt is initialized");
   return 0;
 }
@@ -254,6 +378,7 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) {
 static void dmCleanupMgmt(SMgmtWrapper *pWrapper) {
   dInfo("dnode-mgmt start to clean up");
   SDnode *pDnode = pWrapper->pDnode;
+  dmStopUdfd(pDnode);
   dmStopWorker(pDnode);
 
   taosWLockLatch(&pDnode->data.latch);

source/dnode/mgmt/interface/inc/dmDef.h

@@ -16,6 +16,7 @@
 #ifndef _TD_DM_DEF_H_
 #define _TD_DM_DEF_H_
 
+#include "uv.h"
 #include "dmLog.h"
 
 #include "cJSON.h"
@@ -142,6 +143,18 @@ typedef struct {
   char desc[TSDB_STEP_DESC_LEN];
 } SStartupInfo;
 
+typedef struct SUdfdData {
+  bool          startCalled;
+  bool          needCleanUp;
+  uv_loop_t     loop;
+  uv_thread_t   thread;
+  uv_barrier_t  barrier;
+  uv_process_t  process;
+  int           spawnErr;
+  int8_t        stopping;
+  uv_pipe_t     ctrlPipe;
+} SUdfdData;
+
 typedef struct SDnode {
   EDndProcType  ptype;
   EDndNodeType  ntype;
@@ -150,6 +163,7 @@ typedef struct SDnode {
   SStartupInfo  startup;
   SDnodeTrans   trans;
   SDnodeData    data;
+  SUdfdData     udfdData;
   TdThreadMutex mutex;
   SMgmtWrapper  wrappers[NODE_END];
 } SDnode;

source/dnode/mgmt/test/sut/src/sut.cpp

@@ -89,7 +89,6 @@ int32_t Testbase::SendShowReq(int8_t showType, const char *tb, const char* db) {
   }
 
   SRetrieveTableReq retrieveReq = {0};
-  retrieveReq.type = showType;
   strcpy(retrieveReq.db, db);
   strcpy(retrieveReq.tb, tb);
 

source/dnode/mnode/impl/inc/mndDef.h

@@ -391,7 +391,6 @@ typedef struct {
   int16_t        numOfColumns;
   int32_t        rowSize;
   int32_t        numOfRows;
-  int32_t        payloadLen;
   void*          pIter;
   SMnode*        pMnode;
   STableMetaRsp* pMeta;

source/dnode/mnode/impl/src/mndDb.c

@@ -851,6 +851,8 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) {
   cfgRsp.cacheLastRow = pDb->cfg.cacheLastRow;
   cfgRsp.streamMode = pDb->cfg.streamMode;
   cfgRsp.singleSTable = pDb->cfg.singleSTable;
+  cfgRsp.numOfRetensions = pDb->cfg.numOfRetensions;
+  cfgRsp.pRetensions = pDb->cfg.pRetensions;
 
   int32_t contLen = tSerializeSDbCfgRsp(NULL, 0, &cfgRsp);
   void   *pRsp = rpcMallocCont(contLen);

source/dnode/mnode/impl/src/mndFunc.c

@@ -427,7 +427,6 @@ static int32_t mndProcessRetrieveFuncReq(SNodeMsg *pReq) {
 
     SFuncObj *pFunc = mndAcquireFunc(pMnode, funcName);
     if (pFunc == NULL) {
-      terrno = TSDB_CODE_MND_INVALID_FUNC;
       goto RETRIEVE_FUNC_OVER;
     }
 
@@ -439,21 +438,26 @@ static int32_t mndProcessRetrieveFuncReq(SNodeMsg *pReq) {
     funcInfo.outputLen = pFunc->outputLen;
     funcInfo.bufSize = pFunc->bufSize;
     funcInfo.signature = pFunc->signature;
-    funcInfo.commentSize = pFunc->commentSize;
-    funcInfo.codeSize = pFunc->codeSize;
-    funcInfo.pCode = taosMemoryCalloc(1, funcInfo.codeSize);
-    if (funcInfo.pCode == NULL) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      goto RETRIEVE_FUNC_OVER;
-    }
-    memcpy(funcInfo.pCode, pFunc->pCode, pFunc->codeSize);
-    if (funcInfo.commentSize > 0) {
-      funcInfo.pComment = taosMemoryCalloc(1, funcInfo.commentSize);
-      if (funcInfo.pComment == NULL) {
+    if (retrieveReq.ignoreCodeComment) {
+      funcInfo.commentSize = 0;
+      funcInfo.codeSize = 0;
+    } else {
+      funcInfo.commentSize = pFunc->commentSize;
+      funcInfo.codeSize = pFunc->codeSize;
+      funcInfo.pCode = taosMemoryCalloc(1, funcInfo.codeSize);
+      if (funcInfo.pCode == NULL) {
         terrno = TSDB_CODE_OUT_OF_MEMORY;
         goto RETRIEVE_FUNC_OVER;
       }
-      memcpy(funcInfo.pComment, pFunc->pComment, pFunc->commentSize);
+      memcpy(funcInfo.pCode, pFunc->pCode, pFunc->codeSize);
+      if (funcInfo.commentSize > 0) {
+        funcInfo.pComment = taosMemoryCalloc(1, funcInfo.commentSize);
+        if (funcInfo.pComment == NULL) {
+          terrno = TSDB_CODE_OUT_OF_MEMORY;
+          goto RETRIEVE_FUNC_OVER;
+        }
+        memcpy(funcInfo.pComment, pFunc->pComment, pFunc->commentSize);
+      }
     }
     taosArrayPush(retrieveRsp.pFuncInfos, &funcInfo);
     mndReleaseFunc(pMnode, pFunc);
@@ -518,11 +522,16 @@ static int32_t mndRetrieveFuncs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB
     SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, (const char *)b1, false);
 
-    char *b2 = taosMemoryCalloc(1, pShow->bytes[cols]);
-    STR_WITH_MAXSIZE_TO_VARSTR(b2, pFunc->pComment, pShow->bytes[cols]);
+    if (pFunc->pComment) {
+      char *b2 = taosMemoryCalloc(1, pShow->bytes[cols]);
+      STR_WITH_MAXSIZE_TO_VARSTR(b2, pFunc->pComment, pShow->bytes[cols]);
 
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char *)b2, false);
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      colDataAppend(pColInfo, numOfRows, (const char *)b2, false);
+    } else {
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      colDataAppend(pColInfo, numOfRows, NULL, true);
+    }
 
     int32_t isAgg = (pFunc->funcType == TSDB_FUNC_TYPE_AGGREGATE) ? 1 : 0;
 
@@ -556,4 +565,4 @@ static int32_t mndRetrieveFuncs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB
 static void mndCancelGetNextFunc(SMnode *pMnode, void *pIter) {
   SSdb *pSdb = pMnode->pSdb;
   sdbCancelFetch(pSdb, pIter);
-}
+}
\ No newline at end of file

source/dnode/mnode/impl/src/mndInfoSchema.c

@@ -98,7 +98,7 @@ static const SInfosTableSchema userFuncSchema[] = {
     {.name = "name", .bytes = TSDB_FUNC_NAME_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "comment", .bytes = PATH_MAX - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "aggregate", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "comment", .bytes = TSDB_TYPE_STR_MAX_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
+    {.name = "output_type", .bytes = TSDB_TYPE_STR_MAX_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
     {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
     {.name = "code_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "bufsize", .bytes = 4, .type = TSDB_DATA_TYPE_INT},

source/dnode/mnode/impl/src/mndShow.c

@@ -18,7 +18,7 @@
 
 #define SHOW_STEP_SIZE 100
 
-static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowReq *pReq);
+static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq);
 static void      mndFreeShowObj(SShowObj *pShow);
 static SShowObj *mndAcquireShowObj(SMnode *pMnode, int64_t showId);
 static void      mndReleaseShowObj(SShowObj *pShow, bool forceRemove);
@@ -47,18 +47,80 @@ void mndCleanupShow(SMnode *pMnode) {
   }
 }
 
-static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowReq *pReq) {
+static int32_t convertToRetrieveType(char* name, int32_t len) {
+  int32_t type = -1;
+
+  if (strncasecmp(name, TSDB_INS_TABLE_DNODES, len) == 0) {
+    type = TSDB_MGMT_TABLE_DNODE;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_MNODES, len) == 0) {
+    type = TSDB_MGMT_TABLE_MNODE;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_MODULES, len) == 0) {
+    type = TSDB_MGMT_TABLE_MODULE;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_QNODES, len) == 0) {
+    type = TSDB_MGMT_TABLE_QNODE;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_BNODES, len) == 0) {
+    type = TSDB_MGMT_TABLE_BNODE;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_SNODES, len) == 0) {
+    type = TSDB_MGMT_TABLE_SNODE;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_CLUSTER, len) == 0) {
+    type = TSDB_MGMT_TABLE_CLUSTER;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_DATABASES, len) == 0) {
+    type = TSDB_MGMT_TABLE_DB;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_FUNCTIONS, len) == 0) {
+    type = TSDB_MGMT_TABLE_FUNC;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_INDEXES, len) == 0) {
+    //    type = TSDB_MGMT_TABLE_INDEX;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, len) == 0) {
+    type = TSDB_MGMT_TABLE_STB;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STREAMS, len) == 0) {
+    type = TSDB_MGMT_TABLE_STREAMS;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, len) == 0) {
+    type = TSDB_MGMT_TABLE_TABLE;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, len) == 0) {
+    //    type = TSDB_MGMT_TABLE_DIST;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_USERS, len) == 0) {
+    type = TSDB_MGMT_TABLE_USER;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_LICENCES, len) == 0) {
+    type = TSDB_MGMT_TABLE_GRANTS;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_VGROUPS, len) == 0) {
+    type = TSDB_MGMT_TABLE_VGROUP;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_TOPICS, len) == 0) {
+    type = TSDB_MGMT_TABLE_TOPICS;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_CONSUMERS, len) == 0) {
+    type = TSDB_MGMT_TABLE_CONSUMERS;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_SUBSCRIBES, len) == 0) {
+    type = TSDB_MGMT_TABLE_SUBSCRIBES;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_TRANS, len) == 0) {
+    type = TSDB_MGMT_TABLE_TRANS;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_SMAS, len) == 0) {
+    type = TSDB_MGMT_TABLE_SMAS;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_CONFIGS, len) == 0) {
+    type = TSDB_MGMT_TABLE_CONFIGS;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_CONNS, len) == 0) {
+    type = TSDB_MGMT_TABLE_CONNS;
+  } else if (strncasecmp(name, TSDB_INS_TABLE_QUERIES, len) == 0) {
+    type = TSDB_MGMT_TABLE_QUERIES;
+  }  else if (strncasecmp(name, TSDB_INS_TABLE_VNODES, len) == 0) {
+    type = TSDB_MGMT_TABLE_VNODES;
+  } else {
+//    ASSERT(0);
+  }
+
+  return type;
+}
+
+static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq) {
   SShowMgmt *pMgmt = &pMnode->showMgmt;
 
   int64_t showId = atomic_add_fetch_64(&pMgmt->showId, 1);
   if (showId == 0) atomic_add_fetch_64(&pMgmt->showId, 1);
 
-  int32_t  size = sizeof(SShowObj) + pReq->payloadLen;
+  int32_t  size = sizeof(SShowObj);
+
   SShowObj showObj = {0};
-  showObj.id = showId;
+  showObj.id     = showId;
   showObj.pMnode = pMnode;
-  showObj.type = pReq->type;
-  showObj.payloadLen = pReq->payloadLen;
+  showObj.type   = convertToRetrieveType(pReq->tb, tListLen(pReq->tb));
   memcpy(showObj.db, pReq->db, TSDB_DB_FNAME_LEN);
 
   int32_t   keepTime = tsShellActivityTimer * 6 * 1000;
@@ -127,10 +189,6 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) {
   }
 
   if (retrieveReq.showId == 0) {
-    SShowReq req = {0};
-    req.type = retrieveReq.type;
-    strncpy(req.db, retrieveReq.db, tListLen(req.db));
-
     STableMetaRsp *pMeta = (STableMetaRsp *)taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb) + 1);
     if (pMeta == NULL) {
       terrno = TSDB_CODE_MND_INVALID_INFOS_TBL;
@@ -138,7 +196,7 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) {
       return -1;
     }
 
-    pShow = mndCreateShowObj(pMnode, &req);
+    pShow = mndCreateShowObj(pMnode, &retrieveReq);
     if (pShow == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       mError("failed to process show-meta req since %s", terrstr());

source/dnode/mnode/impl/src/mndStb.c

@@ -20,6 +20,7 @@
 #include "mndInfoSchema.h"
 #include "mndMnode.h"
 #include "mndPerfSchema.h"
+#include "mndScheduler.h"
 #include "mndShow.h"
 #include "mndTrans.h"
 #include "mndUser.h"
@@ -443,6 +444,25 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
     for (int32_t f = 0; f < pRSmaParam->nFuncIds; ++f) {
       *(pRSmaParam->pFuncIds + f) = pStb->aggregationMethod;
     }
+    if (pStb->ast1Len > 0) {
+      if (mndConvertRSmaTask(pStb->pAst1, 0, 0, &pRSmaParam->qmsg1, &pRSmaParam->qmsg1Len) != TSDB_CODE_SUCCESS) {
+        taosMemoryFreeClear(pRSmaParam->pFuncIds);
+        taosMemoryFreeClear(req.stbCfg.pRSmaParam);
+        taosMemoryFreeClear(req.stbCfg.pSchema);
+        return NULL;
+      }
+    }
+    if (pStb->ast2Len > 0) {
+      int32_t qmsgLen2 = 0;
+      if (mndConvertRSmaTask(pStb->pAst2, 0, 0, &pRSmaParam->qmsg2, &pRSmaParam->qmsg2Len) != TSDB_CODE_SUCCESS) {
+        taosMemoryFreeClear(pRSmaParam->pFuncIds);
+        taosMemoryFreeClear(pRSmaParam->qmsg1);
+        taosMemoryFreeClear(req.stbCfg.pRSmaParam);
+        taosMemoryFreeClear(req.stbCfg.pSchema);
+        return NULL;
+      }
+    }
+
     req.stbCfg.pRSmaParam = pRSmaParam;
   }
 
@@ -451,6 +471,8 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
   if (pHead == NULL) {
     if (pRSmaParam) {
       taosMemoryFreeClear(pRSmaParam->pFuncIds);
+      taosMemoryFreeClear(pRSmaParam->qmsg1);
+      taosMemoryFreeClear(pRSmaParam->qmsg2);
       taosMemoryFreeClear(pRSmaParam);
     }
     taosMemoryFreeClear(req.stbCfg.pSchema);
@@ -467,6 +489,8 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
   *pContLen = contLen;
   if (pRSmaParam) {
     taosMemoryFreeClear(pRSmaParam->pFuncIds);
+    taosMemoryFreeClear(pRSmaParam->qmsg1);
+    taosMemoryFreeClear(pRSmaParam->qmsg2);
     taosMemoryFreeClear(pRSmaParam);
   }
   taosMemoryFreeClear(req.stbCfg.pSchema);

source/dnode/mnode/impl/test/func/func.cpp

@@ -240,7 +240,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_MND_FUNC_NOT_EXIST);
   }
 
   {
@@ -371,7 +371,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
 
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_MND_FUNC_NOT_EXIST);
   }
 }
 

source/dnode/mnode/impl/test/show/show.cpp

@@ -36,7 +36,7 @@ TEST_F(MndTestShow, 01_ShowMsg_InvalidMsgMax) {
 
   SRpcMsg* pRsp = test.SendReq(TDMT_MND_SYSTABLE_RETRIEVE, pReq, contLen);
   ASSERT_NE(pRsp, nullptr);
-  ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_MSG);
+  ASSERT_NE(pRsp->code, 0);
 }
 
 TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) {
@@ -50,7 +50,7 @@ TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) {
 
   SRpcMsg* pRsp = test.SendReq(TDMT_MND_SYSTABLE_RETRIEVE, pReq, contLen);
   ASSERT_NE(pRsp, nullptr);
-  ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_MSG);
+  ASSERT_NE(pRsp->code, 0);
 }
 
 TEST_F(MndTestShow, 03_ShowMsg_Conn) {

source/dnode/vnode/src/inc/tsdb.h

@@ -391,6 +391,7 @@ struct SReadH {
 #define TSDB_READ_REPO_ID(rh)   REPO_ID(TSDB_READ_REPO(rh))
 #define TSDB_READ_FSET(rh)      (&((rh)->rSet))
 #define TSDB_READ_TABLE(rh)     ((rh)->pTable)
+#define TSDB_READ_TABLE_UID(rh) ((rh)->pTable->uid)
 #define TSDB_READ_HEAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_HEAD)
 #define TSDB_READ_DATA_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_DATA)
 #define TSDB_READ_LAST_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_LAST)

source/dnode/vnode/src/meta/metaTDBImpl.c

@@ -626,14 +626,8 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) {
 #ifdef META_TDB_SMA_TEST
   STSmaWrapper *pSW = NULL;
 
-  pSW = taosMemoryCalloc(1, sizeof(*pSW));
-  if (pSW == NULL) {
-    return NULL;
-  }
-
   SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid);
   if (pCur == NULL) {
-    taosMemoryFree(pSW);
     return NULL;
   }
 
@@ -653,6 +647,12 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) {
         continue;
       }
 
+      if ((pSW == NULL) && ((pSW = taosMemoryCalloc(1, sizeof(*pSW))) == NULL)) {
+        TDB_FREE(pSmaVal);
+        metaCloseSmaCursor(pCur);
+        return NULL;
+      }
+
       ++pSW->number;
       STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma));
       if (tptr == NULL) {

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -3260,6 +3260,9 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SColumnDat
     return TSDB_CODE_SUCCESS;
   }
 
+  tsdbDebug("vgId:%d succeed to load block statis part for uid %" PRIu64, REPO_ID(pHandle->pTsdb),
+            TSDB_READ_TABLE_UID(&pHandle->rhelper));
+
   int16_t* colIds = pHandle->defaultLoadColumn->pData;
 
   size_t numOfCols = QH_GET_NUM_OF_COLS(pHandle);

source/dnode/vnode/src/tsdb/tsdbReadImpl.c

@@ -322,15 +322,18 @@ int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) {
   ASSERT(pBlock->numOfSubBlocks <= 1);
 
   if (!pBlock->aggrStat) {
+    tsdbDebug("vgId:%d no need to load block statis part for uid %" PRIu64 " since not exist", REPO_ID(pReadh->pRepo),
+              TSDB_READ_TABLE_UID(pReadh));
     return TSDB_STATIS_NONE;
   }
 
   SDFile *pDFileAggr = pBlock->last ? TSDB_READ_SMAL_FILE(pReadh) : TSDB_READ_SMAD_FILE(pReadh);
 
   if (tsdbSeekDFile(pDFileAggr, pBlock->aggrOffset, SEEK_SET) < 0) {
-    tsdbError("vgId:%d failed to load block aggr part while seek file %s to offset %" PRIu64 " since %s",
-              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset,
-              tstrerror(terrno));
+    tsdbError("vgId:%d failed to load block statis part for uid %" PRIu64 " while seek file %s to offset %" PRIu64
+              " since %s",
+              TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr),
+              (uint64_t)pBlock->aggrOffset, tstrerror(terrno));
     return -1;
   }
 
@@ -339,25 +342,28 @@ int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) {
 
   int64_t nreadAggr = tsdbReadDFile(pDFileAggr, (void *)(pReadh->pAggrBlkData), sizeAggr);
   if (nreadAggr < 0) {
-    tsdbError("vgId:%d failed to load block aggr part while read file %s since %s, offset:%" PRIu64 " len :%" PRIzu,
-              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), tstrerror(terrno),
-              (uint64_t)pBlock->aggrOffset, sizeAggr);
+    tsdbError("vgId:%d failed to load block statis part for uid %" PRIu64
+              " while read file %s since %s, offset:%" PRIu64 " len :%" PRIzu,
+              TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr),
+              tstrerror(terrno), (uint64_t)pBlock->aggrOffset, sizeAggr);
     return -1;
   }
 
   if (nreadAggr < sizeAggr) {
     terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-    tsdbError("vgId:%d block aggr part in file %s is corrupted, offset:%" PRIu64 " expected bytes:%" PRIzu
-              " read bytes: %" PRId64,
-              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset, sizeAggr,
-              nreadAggr);
+    tsdbError("vgId:%d block statis part for uid %" PRIu64 " in file %s is corrupted, offset:%" PRIu64
+              " expected bytes:%" PRIzu " read bytes: %" PRId64,
+              TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr),
+              (uint64_t)pBlock->aggrOffset, sizeAggr, nreadAggr);
     return -1;
   }
 
   if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pAggrBlkData), (uint32_t)sizeAggr)) {
     terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-    tsdbError("vgId:%d block aggr part in file %s is corrupted since wrong checksum, offset:%" PRIu64 " len :%" PRIzu,
-              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset, sizeAggr);
+    tsdbError("vgId:%d block statis part for uid %" PRIu64
+              "in file %s is corrupted since wrong checksum, offset:%" PRIu64 " len :%" PRIzu,
+              TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr),
+              (uint64_t)pBlock->aggrOffset, sizeAggr);
     return -1;
   }
   return 0;
@@ -367,7 +373,7 @@ static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) {
   ASSERT(pBlock->numOfSubBlocks <= 1);
   SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh);
   if (tsdbSeekDFile(pDFile, pBlock->offset, SEEK_SET) < 0) {
-    tsdbError("vgId:%d failed to load block statis part while seek file %s to offset %" PRId64 " since %s",
+    tsdbError("vgId:%d failed to load block head part while seek file %s to offset %" PRId64 " since %s",
               TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tstrerror(terrno));
     return -1;
   }
@@ -377,14 +383,14 @@ static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) {
 
   int64_t nread = tsdbReadDFile(pDFile, (void *)(pReadh->pBlkData), size);
   if (nread < 0) {
-    tsdbError("vgId:%d failed to load block statis part while read file %s since %s, offset:%" PRId64 " len :%" PRIzu,
+    tsdbError("vgId:%d failed to load block head part while read file %s since %s, offset:%" PRId64 " len :%" PRIzu,
               TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), (int64_t)pBlock->offset, size);
     return -1;
   }
 
   if (nread < size) {
     terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-    tsdbError("vgId:%d block statis part in file %s is corrupted, offset:%" PRId64 " expected bytes:%" PRIzu
+    tsdbError("vgId:%d block head part in file %s is corrupted, offset:%" PRId64 " expected bytes:%" PRIzu
               " read bytes: %" PRId64,
               TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size, nread);
     return -1;
@@ -392,7 +398,7 @@ static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) {
 
   if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pBlkData), (uint32_t)size)) {
     terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-    tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%" PRIzu,
+    tsdbError("vgId:%d block head part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%" PRIzu,
               TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size);
     return -1;
   }
@@ -546,7 +552,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
   int32_t tsize = (int32_t)tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer);
   if (!taosCheckChecksumWhole((uint8_t *)TSDB_READ_BUF(pReadh), tsize)) {
     terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-    tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%d",
+    tsdbError("vgId:%d block head part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%d",
               TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tsize);
     return -1;
   }

source/dnode/vnode/src/tsdb/tsdbSma.c

@@ -643,6 +643,7 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers
   SSubmitMsgIter msgIter = {0};
   SSubmitBlk    *pBlock = NULL;
   SInterval      interval = {0};
+  TSKEY          lastWinSKey = INT64_MIN;
 
   if (tInitSubmitMsgIter(pMsg, &msgIter) != TSDB_CODE_SUCCESS) {
     return TSDB_CODE_FAILED;
@@ -657,7 +658,7 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers
 
     SSubmitBlkIter blkIter = {0};
     if (tInitSubmitBlkIter(pBlock, &blkIter) != TSDB_CODE_SUCCESS) {
-      tdFreeTSmaWrapper(pSW);
+      pSW = tdFreeTSmaWrapper(pSW);
       break;
     }
 
@@ -667,31 +668,37 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers
         tdFreeTSmaWrapper(pSW);
         break;
       }
-      if (pSW == NULL) {
+      if (!pSW || (pTSma->tableUid != pBlock->suid)) {
+        if (pSW) {
+          pSW = tdFreeTSmaWrapper(pSW);
+        }
         if ((pSW = metaGetSmaInfoByTable(REPO_META(pTsdb), pBlock->suid)) == NULL) {
           break;
         }
         if ((pSW->number) <= 0 || (pSW->tSma == NULL)) {
-          tdFreeTSmaWrapper(pSW);
+          pSW = tdFreeTSmaWrapper(pSW);
           break;
         }
+
         pTSma = pSW->tSma;
-      }
 
-      interval.interval = pTSma->interval;
-      interval.intervalUnit = pTSma->intervalUnit;
-      interval.offset = pTSma->offset;
-      interval.precision = REPO_CFG(pTsdb)->precision;
-      interval.sliding = pTSma->sliding;
-      interval.slidingUnit = pTSma->slidingUnit;
+        interval.interval = pTSma->interval;
+        interval.intervalUnit = pTSma->intervalUnit;
+        interval.offset = pTSma->offset;
+        interval.precision = REPO_CFG(pTsdb)->precision;
+        interval.sliding = pTSma->sliding;
+        interval.slidingUnit = pTSma->slidingUnit;
+      }
 
       TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision);
 
-      tsdbSetExpiredWindow(pTsdb, pItemsHash, pTSma->indexUid, winSKey, version);
-
-      // TODO: release only when suid changes.
-      tdDestroyTSmaWrapper(pSW);
-      taosMemoryFreeClear(pSW);
+      if (lastWinSKey != winSKey) {
+        lastWinSKey = winSKey;
+        tsdbSetExpiredWindow(pTsdb, pItemsHash, pTSma->indexUid, winSKey, version);
+      } else {
+        tsdbDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window ignore as duplicated",
+                  REPO_ID(pTsdb), pTSma->indexUid, winSKey);
+      }
     }
   }
 

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -215,10 +215,21 @@ static int vnodeProcessCreateStbReq(SVnode *pVnode, void *pReq) {
     return -1;
   }
 
+  // TODO: remove the debug log
+  SRSmaParam *param = vCreateTbReq.stbCfg.pRSmaParam;
+  if (param) {
+    printf("qmsg1 len = %d, body = %s\n", param->qmsg1 ? (int32_t)strlen(param->qmsg1) : 0,
+           param->qmsg1 ? param->qmsg1 : "");
+    printf("qmsg1 len = %d, body = %s\n", param->qmsg2 ? (int32_t)strlen(param->qmsg2) : 0,
+           param->qmsg2 ? param->qmsg2 : "");
+  }
+
   taosMemoryFree(vCreateTbReq.stbCfg.pSchema);
   taosMemoryFree(vCreateTbReq.stbCfg.pTagSchema);
   if (vCreateTbReq.stbCfg.pRSmaParam) {
     taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->pFuncIds);
+    taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->qmsg1);
+    taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->qmsg2);
     taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam);
   }
   taosMemoryFree(vCreateTbReq.name);

source/libs/catalog/src/catalog.c

@@ -643,6 +643,49 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmt
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char *funcName, SFuncInfo **out) {
+  char *msg = NULL;
+  int32_t msgLen = 0;
+
+  ctgDebug("try to get udf info from mnode, funcName:%s", funcName);
+
+  int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)]((void *)funcName, &msg, 0, &msgLen);
+  if (code) {
+    ctgError("Build get udf msg failed, code:%x, db:%s", code, funcName);
+    CTG_ERR_RET(code);
+  }
+  
+  SRpcMsg rpcMsg = {
+      .msgType = TDMT_MND_RETRIEVE_FUNC,
+      .pCont   = msg,
+      .contLen = msgLen,
+  };
+
+  SRpcMsg rpcRsp = {0};
+
+  rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp);
+  if (TSDB_CODE_SUCCESS != rpcRsp.code) {
+    if (TSDB_CODE_MND_FUNC_NOT_EXIST == rpcRsp.code) {
+      ctgDebug("funcName %s not exist in mnode", funcName);
+      taosMemoryFreeClear(*out);
+      CTG_RET(TSDB_CODE_SUCCESS);
+    }
+    
+    ctgError("error rsp for get udf, error:%s, funcName:%s", tstrerror(rpcRsp.code), funcName);
+    CTG_ERR_RET(rpcRsp.code);
+  }
+
+  code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)](*out, rpcRsp.pCont, rpcRsp.contLen);
+  if (code) {
+    ctgError("Process get udf rsp failed, code:%x, funcName:%s", code, funcName);
+    CTG_ERR_RET(code);
+  }
+
+  ctgDebug("Got udf from mnode, funcName:%s", funcName);
+
+  return TSDB_CODE_SUCCESS;
+}
+
 
 int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) {
   if (NULL == pCtg->dbCache) {
@@ -2811,6 +2854,30 @@ int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps,
   CTG_API_LEAVE(ctgGetIndexInfoFromMnode(pCtg, pRpc, pMgmtEps, indexName, pInfo));
 }
 
+int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo) {
+  CTG_API_ENTER();
+  
+  if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == funcName || NULL == pInfo) {
+    CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
+  }
+
+  int32_t code = 0;
+  *pInfo = taosMemoryMalloc(sizeof(SFuncInfo));
+  if (NULL == *pInfo) {
+    CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY);
+  }
+
+  CTG_ERR_JRET(ctgGetUdfInfoFromMnode(pCtg, pRpc, pMgmtEps, funcName, pInfo));
+  
+_return:
+
+  if (code) {
+    taosMemoryFreeClear(*pInfo);    
+  }
+  
+  CTG_API_LEAVE(code);
+}
+
 
 void catalogDestroy(void) {
   qInfo("start to destroy catalog");

source/libs/executor/inc/executorimpl.h

@@ -382,7 +382,7 @@ typedef struct SSysTableScanInfo {
   void*   pCur;        // cursor for iterate the local table meta store.
   SArray* scanCols;    // SArray<int16_t> scan column id list
 
-  int32_t             type;  // show type, TODO remove it
+//  int32_t             type;  // show type, TODO remove it
   SName               name;
   SSDataBlock*        pRes;
   int32_t             capacity;
@@ -479,7 +479,7 @@ typedef struct {
 
 typedef struct SGroupbyOperatorInfo {
   SOptrBasicInfo  binfo;
-  SArray*         pGroupCols;
+  SArray*         pGroupCols;     // group by columns, SArray<SColumn>
   SArray*         pGroupColVals;  // current group column values, SArray<SGroupKeys>
   SNode*          pCondition;
   bool            isInit;       // denote if current val is initialized or not
@@ -600,7 +600,6 @@ typedef struct SJoinOperatorInfo {
   int32_t            rightPos;
   SColumnInfo        rightCol;
   SNode             *pOnCondition;
-//  SRspResultInfo     resultInfo;
 } SJoinOperatorInfo;
 
 int32_t operatorDummyOpenFn(SOperatorInfo* pOperator);

source/libs/executor/src/executorimpl.c

@@ -1087,6 +1087,7 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt
     pCtx[i].currentStage = MAIN_SCAN;
 
     SInputColumnInfoData* pInput = &pCtx[i].input;
+    pInput->uid = pBlock->info.uid;
 
     SExprInfo* pOneExpr = &pOperator->pExpr[i];
     for (int32_t j = 0; j < pOneExpr->base.numOfParams; ++j) {
@@ -1101,7 +1102,9 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt
         pInput->pPTS = taosArrayGet(pBlock->pDataBlock, 0); // todo set the correct timestamp column
         ASSERT(pInput->pData[j] != NULL);
       } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) {
-        if (createDummyCol) {
+        // todo avoid case: top(k, 12), 12 is the value parameter.
+        // sum(11), 11 is also the value parameter.
+        if (createDummyCol && pOneExpr->base.numOfParams == 1) {
           code = doCreateConstantValColumnInfo(pInput, pFuncParam, pFuncParam->param.nType, j, pBlock->info.rows);
           if (code != TSDB_CODE_SUCCESS) {
             return code;
@@ -1876,67 +1879,58 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
         }
       }
       pCtx->resDataInfo.interBufSize = env.calcMemSize;
-    } else if (pExpr->pExpr->nodeType == QUERY_NODE_COLUMN || pExpr->pExpr->nodeType == QUERY_NODE_OPERATOR || pExpr->pExpr->nodeType == QUERY_NODE_VALUE) {
-      pCtx->resDataInfo.interBufSize = pFunct->resSchema.bytes; // for simple column, the intermediate buffer needs to hold one element.
+    } else if (pExpr->pExpr->nodeType == QUERY_NODE_COLUMN || pExpr->pExpr->nodeType == QUERY_NODE_OPERATOR ||
+               pExpr->pExpr->nodeType == QUERY_NODE_VALUE) {
+      // for simple column, the intermediate buffer needs to hold one element.
+      pCtx->resDataInfo.interBufSize = pFunct->resSchema.bytes;
     }
 
     pCtx->input.numOfInputCols = pFunct->numOfParams;
     pCtx->input.pData = taosMemoryCalloc(pFunct->numOfParams, POINTER_BYTES);
     pCtx->input.pColumnDataAgg = taosMemoryCalloc(pFunct->numOfParams, POINTER_BYTES);
 
-    pCtx->pTsOutput         = NULL;
+    pCtx->pTsOutput = NULL;
     pCtx->resDataInfo.bytes = pFunct->resSchema.bytes;
-    pCtx->resDataInfo.type  = pFunct->resSchema.type;
-    pCtx->order             = TSDB_ORDER_ASC;
-    pCtx->start.key         = INT64_MIN;
-    pCtx->end.key           = INT64_MIN;
-#if 0
-    for (int32_t j = 0; j < pCtx->numOfParams; ++j) {
-//      int16_t type = pFunct->param[j].nType;
-//      int16_t bytes = pFunct->param[j].nLen;
-
-//      if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
-//        taosVariantCreateFromBinary(&pCtx->param[j], pFunct->param[j].pz, bytes, type);
-//      } else {
-//        taosVariantCreateFromBinary(&pCtx->param[j], (char *)&pFunct->param[j].i, bytes, type);
-//      }
-    }
-
-    // set the order information for top/bottom query
-    int32_t functionId = pCtx->functionId;
-    if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) {
-      int32_t f = getExprFunctionId(&pExpr[0]);
-      assert(f == FUNCTION_TS || f == FUNCTION_TS_DUMMY);
-
-//      pCtx->param[2].i = pQueryAttr->order.order;
-      pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
-      pCtx->param[3].i = functionId;
-      pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT;
-
-//      pCtx->param[1].i = pQueryAttr->order.col.info.colId;
-    } else if (functionId == FUNCTION_INTERP) {
-//      pCtx->param[2].i = (int8_t)pQueryAttr->fillType;
-//      if (pQueryAttr->fillVal != NULL) {
-//        if (isNull((const char *)&pQueryAttr->fillVal[i], pCtx->inputType)) {
-//          pCtx->param[1].nType = TSDB_DATA_TYPE_NULL;
-//        } else {  // todo refactor, taosVariantCreateFromBinary should handle the NULL value
-//          if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) {
-//            taosVariantCreateFromBinary(&pCtx->param[1], (char *)&pQueryAttr->fillVal[i], pCtx->inputBytes, pCtx->inputType);
-//          }
-//        }
+    pCtx->resDataInfo.type = pFunct->resSchema.type;
+    pCtx->order = TSDB_ORDER_ASC;
+    pCtx->start.key = INT64_MIN;
+    pCtx->end.key = INT64_MIN;
+    pCtx->numOfParams = pExpr->base.numOfParams;
+
+    pCtx->param = pFunct->pParam;
+//    for (int32_t j = 0; j < pCtx->numOfParams; ++j) {
+//      // set the order information for top/bottom query
+//      int32_t functionId = pCtx->functionId;
+//      if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) {
+//        int32_t f = getExprFunctionId(&pExpr[0]);
+//        assert(f == FUNCTION_TS || f == FUNCTION_TS_DUMMY);
+//
+//        //      pCtx->param[2].i = pQueryAttr->order.order;
+//        //      pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
+//        //      pCtx->param[3].i = functionId;
+//        //      pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT;
+//
+//        //      pCtx->param[1].i = pQueryAttr->order.col.info.colId;
+//      } else if (functionId == FUNCTION_INTERP) {
+//        //      pCtx->param[2].i = (int8_t)pQueryAttr->fillType;
+//        //      if (pQueryAttr->fillVal != NULL) {
+//        //        if (isNull((const char *)&pQueryAttr->fillVal[i], pCtx->inputType)) {
+//        //          pCtx->param[1].nType = TSDB_DATA_TYPE_NULL;
+//        //        } else {  // todo refactor, taosVariantCreateFromBinary should handle the NULL value
+//        //          if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) {
+//        //            taosVariantCreateFromBinary(&pCtx->param[1], (char *)&pQueryAttr->fillVal[i], pCtx->inputBytes, pCtx->inputType);
+//        //          }
+//        //        }
+//        //      }
+//      } else if (functionId == FUNCTION_TWA) {
+//        //      pCtx->param[1].i = pQueryAttr->window.skey;
+//        //      pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT;
+//        //      pCtx->param[2].i = pQueryAttr->window.ekey;
+//        //      pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
+//      } else if (functionId == FUNCTION_ARITHM) {
+//        //      pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i);
 //      }
-    } else if (functionId == FUNCTION_TS_COMP) {
-//      pCtx->param[0].i = pQueryAttr->vgId;  //TODO this should be the parameter from client
-      pCtx->param[0].nType = TSDB_DATA_TYPE_BIGINT;
-    } else if (functionId == FUNCTION_TWA) {
-//      pCtx->param[1].i = pQueryAttr->window.skey;
-      pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT;
-//      pCtx->param[2].i = pQueryAttr->window.ekey;
-      pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT;
-    } else if (functionId == FUNCTION_ARITHM) {
-//      pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i);
-    }
-#endif
+//    }
   }
 
   for (int32_t i = 1; i < numOfOutput; ++i) {
@@ -1955,7 +1949,7 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
 
   for (int32_t i = 0; i < numOfOutput; ++i) {
     for (int32_t j = 0; j < pCtx[i].numOfParams; ++j) {
-      taosVariantDestroy(&pCtx[i].param[j]);
+      taosVariantDestroy(&pCtx[i].param[j].param);
     }
 
     taosVariantDestroy(&pCtx[i].tag);
@@ -6487,9 +6481,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
 
       int32_t     numOfCols = 0;
       tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId);
-      if (pDataReader == NULL) {
-        return NULL;
-      }
 
       SArray* pColList = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols);
       SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc);

source/libs/executor/src/scanoperator.c

@@ -319,7 +319,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
   SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
 
   // The read handle is not initialized yet, since no qualified tables exists
-  if (pTableScanInfo->dataReader == NULL) {
+  if (pTableScanInfo->dataReader == NULL || pOperator->status == OP_EXEC_DONE) {
     return NULL;
   }
 
@@ -375,10 +375,9 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
   return p;
 }
 
-SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput,
-                                           int32_t dataLoadFlag, int32_t repeatTime, int32_t reverseTime,
-                                           SArray* pColMatchInfo, SSDataBlock* pResBlock, SNode* pCondition,
-                                           SInterval* pInterval, double sampleRatio, SExecTaskInfo* pTaskInfo) {
+SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, int32_t order, int32_t numOfOutput, int32_t dataLoadFlag,
+                                           int32_t repeatTime, int32_t reverseTime, SArray* pColMatchInfo, SSDataBlock* pResBlock,
+                                           SNode* pCondition, SInterval* pInterval, double sampleRatio, SExecTaskInfo* pTaskInfo) {
   assert(repeatTime > 0);
 
   STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
@@ -391,19 +390,19 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order,
     return NULL;
   }
 
-  pInfo->interval = *pInterval;
-  pInfo->sampleRatio = sampleRatio;
-  pInfo->dataBlockLoadFlag = dataLoadFlag;
-  pInfo->pResBlock = pResBlock;
-  pInfo->pFilterNode = pCondition;
-  pInfo->dataReader = pTsdbReadHandle;
-  pInfo->times = repeatTime;
-  pInfo->reverseTimes = reverseTime;
-  pInfo->order = order;
-  pInfo->current = 0;
-  pInfo->scanFlag = MAIN_SCAN;
-  pInfo->pColMatchInfo = pColMatchInfo;
-  pOperator->name = "TableScanOperator";
+  pInfo->interval         = *pInterval;
+  pInfo->sampleRatio      = sampleRatio;
+  pInfo->dataBlockLoadFlag= dataLoadFlag;
+  pInfo->pResBlock        = pResBlock;
+  pInfo->pFilterNode      = pCondition;
+  pInfo->dataReader       = pDataReader;
+  pInfo->times            = repeatTime;
+  pInfo->reverseTimes     = reverseTime;
+  pInfo->order            = order;
+  pInfo->current          = 0;
+  pInfo->scanFlag         = MAIN_SCAN;
+  pInfo->pColMatchInfo    = pColMatchInfo;
+  pOperator->name         = "TableScanOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN;
   pOperator->blockingOptr = false;
   pOperator->status = OP_NOT_OPENED;
@@ -677,7 +676,8 @@ static void destroySysScanOperator(void* param, int32_t numOfOutput) {
   tsem_destroy(&pInfo->ready);
   blockDataDestroy(pInfo->pRes);
 
-  if (pInfo->type == TSDB_MGMT_TABLE_TABLE) {
+  const char* name = tNameGetTableName(&pInfo->name);
+  if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) {
     metaCloseTbCursor(pInfo->pCur);
   }
 }
@@ -812,7 +812,8 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) {
   SSysTableScanInfo* pInfo = pOperator->info;
 
   // retrieve local table list info from vnode
-  if (pInfo->type == TSDB_MGMT_TABLE_TABLE) {
+  const char* name = tNameGetTableName(&pInfo->name);
+  if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) {
     if (pInfo->pCur == NULL) {
       pInfo->pCur = metaOpenTbCursor(pInfo->readHandle);
     }
@@ -864,8 +865,6 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) {
 
     while (1) {
       int64_t startTs = taosGetTimestampUs();
-
-      pInfo->req.type = pInfo->type;
       strncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb));
 
       if (pInfo->showRewrite) {
@@ -947,68 +946,9 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB
   pInfo->pCondition = pCondition;
   pInfo->scanCols = colList;
 
-  // TODO remove it
-  int32_t     tableType = 0;
-  const char* name = tNameGetTableName(pName);
-  if (strncasecmp(name, TSDB_INS_TABLE_DNODES, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_DNODE;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_MNODES, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_MNODE;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_MODULES, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_MODULE;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_QNODES, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_QNODE;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_BNODES, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_BNODE;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_SNODES, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_SNODE;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_CLUSTER, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_CLUSTER;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_DATABASES, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_DB;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_FUNCTIONS, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_FUNC;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_INDEXES, tListLen(pName->tname)) == 0) {
-    //    tableType = TSDB_MGMT_TABLE_INDEX;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_STB;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STREAMS, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_STREAMS;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_TABLE;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, tListLen(pName->tname)) == 0) {
-    //    tableType = TSDB_MGMT_TABLE_DIST;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_USERS, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_USER;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_LICENCES, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_GRANTS;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_VGROUPS, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_VGROUP;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_TOPICS, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_TOPICS;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_CONSUMERS, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_CONSUMERS;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_SUBSCRIBES, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_SUBSCRIBES;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_TRANS, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_TRANS;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_SMAS, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_SMAS;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_CONFIGS, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_CONFIGS;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_CONNS, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_CONNS;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_QUERIES, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_QUERIES;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_VNODES, tListLen(pName->tname)) == 0) {
-    tableType = TSDB_MGMT_TABLE_VNODES;
-  } else {
-    ASSERT(0);
-  }
-
   tNameAssign(&pInfo->name, pName);
-  pInfo->type = tableType;
-  if (pInfo->type == TSDB_MGMT_TABLE_TABLE) {
+  const char* name = tNameGetTableName(&pInfo->name);
+  if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) {
     pInfo->readHandle = pSysTableReadHandle;
     blockDataEnsureCapacity(pInfo->pRes, pInfo->capacity);
   } else {

source/libs/function/inc/builtinsimpl.h

@@ -58,6 +58,9 @@ bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 int32_t firstFunction(SqlFunctionCtx *pCtx);
 int32_t lastFunction(SqlFunctionCtx *pCtx);
 
+bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv);
+int32_t topFunction(SqlFunctionCtx *pCtx);
+
 #ifdef __cplusplus
 }
 #endif

source/libs/function/inc/tudf.h

@@ -27,41 +27,31 @@ extern "C" {
 #endif
 
 #define UDF_LISTEN_PIPE_NAME_LEN 32
-#define UDF_LISTEN_PIPE_NAME_PREFIX "udf.sock."
+#define UDF_LISTEN_PIPE_NAME_PREFIX "udfd.sock."
 
 //======================================================================================
 //begin API to taosd and qworker
 
 enum {
   UDFC_CODE_STOPPING = -1,
-  UDFC_CODE_RESTARTING = -2,
   UDFC_CODE_PIPE_READ_ERR = -3,
 };
 
-/*TODO: no api for dnode startudfd/stopudfd*/
-/**
- * start udfd dameon service
- */
-int32_t startUdfd(int32_t dnodeId);
-
-/**
- * stop udfd dameon service
- */
-int32_t stopUdfd(int32_t dnodeId);
+typedef void *UdfcHandle;
+typedef void *UdfcFuncHandle;
 
 /**
  * create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf
  * @return error code
  */
-int32_t createUdfdProxy(int32_t dnodeId);
+int32_t udfcOpen(int32_t dnodeId, UdfcHandle* proxyHandle);
 
 /**
  * destroy udfd proxy
  * @return error code
  */
-int32_t destroyUdfdProxy(int32_t dnodeId);
+int32_t udfcClose(UdfcHandle proxyhandle);
 
-typedef void *UdfHandle;
 
 /**
  * setup udf
@@ -69,7 +59,7 @@ typedef void *UdfHandle;
  * @param handle, out
  * @return error code
  */
-int32_t setupUdf(char udfName[], SEpSet *epSet, UdfHandle *handle);
+int32_t setupUdf(UdfcHandle proxyHandle, char udfName[], SEpSet *epSet, UdfcFuncHandle *handle);
 
 typedef struct SUdfColumnMeta {
   int16_t type;
@@ -116,26 +106,26 @@ typedef struct SUdfInterBuf {
 } SUdfInterBuf;
 
 // output: interBuf
-int32_t callUdfAggInit(UdfHandle handle, SUdfInterBuf *interBuf);
+int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf);
 // input: block, state
 // output: newState
-int32_t callUdfAggProcess(UdfHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState);
+int32_t callUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState);
 // input: interBuf
 // output: resultData
-int32_t callUdfAggFinalize(UdfHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData);
+int32_t callUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData);
 // input: interbuf1, interbuf2
 // output: resultBuf
-int32_t callUdfAggMerge(UdfHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf);
+int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf);
 // input: block
 // output: resultData
-int32_t callUdfScalaProcess(UdfHandle handle, SSDataBlock *block, SSDataBlock *resultData);
+int32_t callUdfScalaProcess(UdfcFuncHandle handle, SSDataBlock *block, SSDataBlock *resultData);
 
 /**
  * tearn down udf
  * @param handle
  * @return
  */
-int32_t teardownUdf(UdfHandle handle);
+int32_t teardownUdf(UdfcFuncHandle handle);
 
 // end API to taosd and qworker
 //=============================================================================================================================

source/libs/function/inc/udfc.h

@@ -30,20 +30,20 @@ typedef struct SUdfInfo {
     char *path;
 } SUdfInfo;
 
-typedef void *UdfHandle;
+typedef void *UdfcFuncHandle;
 
 int32_t createUdfdProxy();
 
 int32_t destroyUdfdProxy();
 
-//int32_t setupUdf(SUdfInfo *udf, int32_t numOfUdfs, UdfHandle *handles);
+//int32_t setupUdf(SUdfInfo *udf, int32_t numOfUdfs, UdfcFuncHandle *handles);
 
-int32_t setupUdf(SUdfInfo* udf, UdfHandle* handle);
+int32_t setupUdf(SUdfInfo* udf, UdfcFuncHandle* handle);
 
-int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SSDataBlock input, char **newstate,
+int32_t callUdf(UdfcFuncHandle handle, int8_t step, char *state, int32_t stateSize, SSDataBlock input, char **newstate,
                 int32_t *newStateSize, SSDataBlock *output);
 
-int32_t teardownUdf(UdfHandle handle);
+int32_t teardownUdf(UdfcFuncHandle handle);
 
 typedef struct SUdfSetupRequest {
     char udfName[16]; //

source/libs/function/src/builtins.c

@@ -193,8 +193,15 @@ static int32_t translateApercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t translateTbnameColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  // pseudo column do not need to check parameters
+  pFunc->node.resType = (SDataType){.bytes = TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR};
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t translateTop(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
-  // todo
+  SDataType* pType = &((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType;
+  pFunc->node.resType = (SDataType){.bytes = pType->bytes, .type = pType->type};
   return TSDB_CODE_SUCCESS;
 }
 
@@ -497,9 +504,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .type = FUNCTION_TYPE_TOP,
     .classification = FUNC_MGT_AGG_FUNC,
     .translateFunc = translateTop,
-    .getEnvFunc   = getMinmaxFuncEnv,
-    .initFunc     = maxFunctionSetup,
-    .processFunc  = maxFunction,
+    .getEnvFunc   = getTopBotFuncEnv,
+    .initFunc     = functionSetup,
+    .processFunc  = topFunction,
     .finalizeFunc = functionFinalize
   },
   {
@@ -876,7 +883,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .name = "tbname",
     .type = FUNCTION_TYPE_TBNAME,
     .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC,
-    .translateFunc = NULL,
+    .translateFunc = translateTbnameColumn,
     .getEnvFunc   = NULL,
     .initFunc     = NULL,
     .sprocessFunc = NULL,
@@ -914,7 +921,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   },
   {
     .name = "_wendts",
-    .type = FUNCTION_TYPE_QENDTS,
+    .type = FUNCTION_TYPE_WENDTS,
     .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC,
     .translateFunc = translateTimePseudoColumn,
     .getEnvFunc   = getTimePseudoFuncEnv,

source/libs/function/src/builtinsimpl.c

@@ -14,10 +14,11 @@
  */
 
 #include "builtinsimpl.h"
-#include "tpercentile.h"
+#include <libs/nodes/querynodes.h>
 #include "querynodes.h"
 #include "taggfunction.h"
 #include "tdatablock.h"
+#include "tpercentile.h"
 
 #define SET_VAL(_info, numOfElem, res)  \
   do {                                  \
@@ -472,17 +473,6 @@ int32_t maxFunction(SqlFunctionCtx *pCtx) {
   return TSDB_CODE_SUCCESS;
 }
 
-typedef struct STopBotRes {
-  int32_t num;
-} STopBotRes;
-
-bool getTopBotFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
-    SColumnNode* pColNode = (SColumnNode*) nodesListGetNode(pFunc->pParameterList, 0);
-  int32_t bytes = pColNode->node.resType.bytes;
-  SValueNode* pkNode = (SValueNode*) nodesListGetNode(pFunc->pParameterList, 1);
-  return true;
-}
-
 typedef struct SStddevRes {
   double  result;
   int64_t count;
@@ -523,7 +513,7 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) {
   switch (type) {
     case TSDB_DATA_TYPE_TINYINT: {
         int8_t* plist = (int8_t*)pCol->pData;
-        for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
+        for (int32_t i = start; i < numOfRows + start; ++i) {
           if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
             continue;
           }
@@ -749,9 +739,9 @@ int32_t percentileFunction(SqlFunctionCtx *pCtx) {
   return TSDB_CODE_SUCCESS;
 }
 
-// TODO set the correct parameter.
 void percentileFinalize(SqlFunctionCtx* pCtx) {
-  double v = 50;//pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64 : pCtx->param[0].dKey;
+  SVariant* pVal = &pCtx->param[1].param;
+  double v = pVal->nType == TSDB_DATA_TYPE_INT ? pVal->i : pVal->d;
 
   SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx);
   SPercentileInfo* ppInfo = (SPercentileInfo *) GET_ROWCELL_INTERBUF(pResInfo);
@@ -1173,3 +1163,130 @@ int32_t diffFunction(SqlFunctionCtx *pCtx) {
   }
 }
 
+typedef struct STopBotResItem {
+  SVariant v;
+  uint64_t uid;        // it is a table uid, used to extract tag data during building of the final result for the tag data
+  struct {
+   int32_t pageId;
+   int32_t offset;
+  } tuplePos;          // tuple data of this chosen row
+} STopBotResItem;
+
+typedef struct STopBotRes {
+  int32_t   num;
+  STopBotResItem *pItems;
+} STopBotRes;
+
+bool getTopBotFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
+  SColumnNode* pColNode = (SColumnNode*) nodesListGetNode(pFunc->pParameterList, 0);
+  int32_t bytes = pColNode->node.resType.bytes;
+  SValueNode* pkNode = (SValueNode*) nodesListGetNode(pFunc->pParameterList, 1);
+
+  pEnv->calcMemSize = sizeof(STopBotRes) + pkNode->datum.i * bytes;
+  return true;
+}
+
+static STopBotRes *getTopBotOutputInfo(SqlFunctionCtx *pCtx) {
+  SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx);
+  STopBotRes* pRes = GET_ROWCELL_INTERBUF(pResInfo);
+  pRes->pItems = (STopBotResItem*)((char*) pRes + sizeof(STopBotRes));
+
+  return pRes;
+}
+
+static void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, uint16_t type, uint64_t uid);
+
+int32_t topFunction(SqlFunctionCtx *pCtx) {
+  int32_t numOfElems = 0;
+
+  STopBotRes *pRes = getTopBotOutputInfo(pCtx);
+  assert(pRes->num >= 0);
+
+//  if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotRes) + POINTER_BYTES * pCtx->param[0].i)) {
+//    buildTopBotStruct(pRes, pCtx);
+//  }
+
+  SInputColumnInfoData* pInput = &pCtx->input;
+  SColumnInfoData* pCol = pInput->pData[0];
+
+  int32_t type = pInput->pData[0]->info.type;
+
+  int32_t start = pInput->startRowIndex;
+  int32_t numOfRows = pInput->numOfRows;
+
+  for (int32_t i = start; i < numOfRows + start; ++i) {
+    if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
+      continue;
+    }
+    numOfElems++;
+
+    char* data = colDataGetData(pCol, i);
+    doAddIntoResult(pRes, pCtx->param[1].param.i, data, type, pInput->uid);
+  }
+
+  // treat the result as only one result
+  SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1);
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t topBotResComparFn(const void *p1, const void *p2, const void *param) {
+  uint16_t type = *(uint16_t *) param;
+
+  STopBotResItem  *val1 = (STopBotResItem *) p1;
+  STopBotResItem  *val2 = (STopBotResItem *) p2;
+
+  if (IS_SIGNED_NUMERIC_TYPE(type)) {
+    if (val1->v.i == val2->v.i) {
+      return 0;
+    }
+
+    return (val1->v.i > val2->v.i) ? 1 : -1;
+  } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
+    if (val1->v.u == val2->v.u) {
+      return 0;
+    }
+
+    return (val1->v.u > val2->v.u) ? 1 : -1;
+  }
+
+  if (val1->v.d == val2->v.d) {
+    return 0;
+  }
+
+  return (val1->v.d > val2->v.d) ? 1 : -1;
+}
+
+void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, uint16_t type, uint64_t uid) {
+  SVariant val = {0};
+  taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type);
+
+  STopBotResItem *pItems = pRes->pItems;
+  assert(pItems != NULL);
+
+  // not full yet
+  if (pRes->num < maxSize) {
+    STopBotResItem* pItem = &pItems[pRes->num];
+    pItem->v   = val;
+    pItem->uid = uid;
+    pItem->tuplePos.pageId = -1;  // todo set the corresponding tuple data in the disk-based buffer
+
+    pRes->num++;
+    taosheapsort((void *) pItem, sizeof(STopBotResItem), pRes->num, (const void *) &type, topBotResComparFn, false);
+  } else { // replace the minimum value in the result
+    if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i > pItems[0].v.i) ||
+        (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u > pItems[0].v.u) ||
+        (IS_FLOAT_TYPE(type) && val.d > pItems[0].v.d)) {
+      STopBotResItem* pItem = &pItems[pRes->num];
+      pItem->v   = val;
+      pItem->uid = uid;
+      pItem->tuplePos.pageId = -1;  // todo set the corresponding tuple data in the disk-based buffer
+
+      taosheapadjust((void *) pItem, sizeof(STopBotResItem), 0, pRes->num - 1, (const void *) &type, topBotResComparFn, NULL, false);
+    }
+  }
+}
+
+void topBotFinalize(SqlFunctionCtx* pCtx) {
+  functionFinalize(pCtx);
+
+}
\ No newline at end of file

source/libs/function/src/functionMgt.c

@@ -29,6 +29,7 @@ typedef struct SFuncMgtService {
 
 typedef struct SUdfInfo {
   SDataType outputDt;
+  int8_t funcType;
 } SUdfInfo;
 
 static SFuncMgtService gFunMgtService;
@@ -52,30 +53,41 @@ static void doInitFunctionTable() {
   gFunMgtService.pUdfTable = NULL;
 }
 
+static int8_t getUdfType(int32_t funcId) {
+  SUdfInfo* pUdf = taosArrayGet(gFunMgtService.pUdfTable, funcId - FUNC_UDF_ID_START_OFFSET_VAL - 1);
+  return pUdf->funcType;
+}
+
 static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) {
+  if (fmIsUserDefinedFunc(funcId)) {
+    return getUdfType(funcId);
+  }
   if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
     return false;
   }
   return FUNC_MGT_TEST_MASK(funcMgtBuiltins[funcId].classification, classification);
 }
 
-static int32_t getUdfId(const char* pFuncName) {
-  // todo: udf by call catalog
-  if (1) {
+static int32_t getUdfId(SFmGetFuncInfoParam* pParam, const char* pFuncName) {
+  SFuncInfo* pInfo = NULL;
+  int32_t code = catalogGetUdfInfo(pParam->pCtg, pParam->pRpc, pParam->pMgmtEps, pFuncName, &pInfo);
+  if (TSDB_CODE_SUCCESS != code || NULL == pInfo) {
     return -1;
   }
   if (NULL == gFunMgtService.pUdfTable) {
     gFunMgtService.pUdfTable = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SUdfInfo));
   }
-  SUdfInfo info = {0}; //todo
+  SUdfInfo info = { .outputDt.type = pInfo->outputType, .outputDt.bytes = pInfo->outputLen, .funcType = pInfo->funcType };
   taosArrayPush(gFunMgtService.pUdfTable, &info);
+  tFreeSFuncInfo(pInfo);
+  taosMemoryFree(pInfo);
   return taosArrayGetSize(gFunMgtService.pUdfTable) + FUNC_UDF_ID_START_OFFSET_VAL;
 }
 
-static int32_t getFuncId(const char* pFuncName) {
+static int32_t getFuncId(SFmGetFuncInfoParam* pParam, const char* pFuncName) {
   void* pVal = taosHashGet(gFunMgtService.pFuncNameHashTable, pFuncName, strlen(pFuncName));
   if (NULL == pVal) {
-    return getUdfId(pFuncName);
+    return getUdfId(pParam, pFuncName);
   }
   return *(int32_t*)pVal;
 }
@@ -91,8 +103,8 @@ int32_t fmFuncMgtInit() {
   return initFunctionCode;
 }
 
-int32_t fmGetFuncInfo(const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType) {
-  *pFuncId = getFuncId(pFuncName);
+int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType) {
+  *pFuncId = getFuncId(pParam, pFuncName);
   if (*pFuncId < 0) {
     return TSDB_CODE_FAILED;
   }

source/libs/function/src/tudf.c

@@ -14,19 +14,15 @@
  */
 #include "uv.h"
 #include "os.h"
-#include "tlog.h"
 #include "tudf.h"
 #include "tudfInt.h"
 #include "tarray.h"
 #include "tdatablock.h"
 
-//TODO: when startup, set thread poll size. add it to cfg
-//TODO: test for udfd restart
-//TODO: udfd restart when exist or aborts
-//TODO: deal with uv task that has been started and then udfd core dumped
 //TODO: network error processing.
 //TODO: add unit test
 //TODO: include all global variable under context struct
+
 /* Copyright (c) 2013, Ben Noordhuis <info@bnoordhuis.nl>
  * The QUEUE is copied from queue.h under libuv
  * */
@@ -125,12 +121,35 @@ enum {
   UV_TASK_DISCONNECT = 2
 };
 
+int64_t gUdfTaskSeqNum = 0;
+typedef struct SUdfdProxy {
+  int32_t dnodeId;
+  uv_barrier_t gUdfInitBarrier;
+
+  uv_loop_t gUdfdLoop;
+  uv_thread_t gUdfLoopThread;
+  uv_async_t gUdfLoopTaskAync;
+
+  uv_async_t gUdfLoopStopAsync;
+
+  uv_mutex_t gUdfTaskQueueMutex;
+  int8_t gUdfcState;
+  QUEUE gUdfTaskQueue;
+  QUEUE gUvProcTaskQueue;
+  //  int8_t gUdfcState = UDFC_STATE_INITAL;
+  //  QUEUE gUdfTaskQueue = {0};
+  //  QUEUE gUvProcTaskQueue = {0};
+} SUdfdProxy;
+
+
 typedef struct SUdfUvSession {
+  SUdfdProxy *udfc;
   int64_t severHandle;
   uv_pipe_t *udfSvcPipe;
 } SUdfUvSession;
 
 typedef struct SClientUvTaskNode {
+  SUdfdProxy *udfc;
   int8_t type;
   int errCode;
 
@@ -169,7 +188,6 @@ typedef struct SClientUdfTask {
     } _teardown;
   };
 
-
 } SClientUdfTask;
 
 typedef struct SClientConnBuf {
@@ -185,34 +203,13 @@ typedef struct SClientUvConn {
   SClientConnBuf readBuf;
 } SClientUvConn;
 
-uv_process_t gUdfdProcess;
-
-uv_barrier_t gUdfInitBarrier;
-
-uv_loop_t gUdfdLoop;
-uv_thread_t gUdfLoopThread;
-uv_async_t gUdfLoopTaskAync;
-
-uv_async_t gUdfLoopStopAsync;
-
-uv_mutex_t gUdfTaskQueueMutex;
-int64_t gUdfTaskSeqNum = 0;
-
 enum {
   UDFC_STATE_INITAL = 0, // initial state
-  UDFC_STATE_STARTNG, // starting after createUdfdProxy
+  UDFC_STATE_STARTNG, // starting after udfcOpen
   UDFC_STATE_READY, // started and begin to receive quests
-  UDFC_STATE_RESTARTING, // udfd abnormal exit. cleaning up and restart.
-  UDFC_STATE_STOPPING, // stopping after destroyUdfdProxy
+  UDFC_STATE_STOPPING, // stopping after udfcClose
   UDFC_STATUS_FINAL, // stopped
 };
-int8_t gUdfcState = UDFC_STATE_INITAL;
-
-//double circular linked list
-
-QUEUE gUdfTaskQueue = {0};
-
-QUEUE gUvProcTaskQueue = {0};
 
 int32_t encodeUdfSetupRequest(void **buf, const SUdfSetupRequest *setup) {
   int32_t len = 0;
@@ -777,13 +774,14 @@ void onUdfClientConnect(uv_connect_t *connect, int status) {
 int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskNode **pUvTask) {
   SClientUvTaskNode *uvTask = taosMemoryCalloc(1, sizeof(SClientUvTaskNode));
   uvTask->type = uvTaskType;
+  uvTask->udfc = task->session->udfc;
 
   if (uvTaskType == UV_TASK_CONNECT) {
   } else if (uvTaskType == UV_TASK_REQ_RSP) {
     uvTask->pipe = task->session->udfSvcPipe;
     SUdfRequest request;
     request.type = task->type;
-    request.seqNum = gUdfTaskSeqNum++;
+    request.seqNum =   atomic_fetch_add_64(&gUdfTaskSeqNum, 1);
 
     if (task->type == UDF_TASK_SETUP) {
       request.setup = task->_setup.req;
@@ -815,11 +813,11 @@ int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskN
 
 int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) {
   debugPrint("%s, %d", "queue uv task", uvTask->type);
-
-  uv_mutex_lock(&gUdfTaskQueueMutex);
-  QUEUE_INSERT_TAIL(&gUdfTaskQueue, &uvTask->recvTaskQueue);
-  uv_mutex_unlock(&gUdfTaskQueueMutex);
-  uv_async_send(&gUdfLoopTaskAync);
+  SUdfdProxy *udfc = uvTask->udfc;
+  uv_mutex_lock(&udfc->gUdfTaskQueueMutex);
+  QUEUE_INSERT_TAIL(&udfc->gUdfTaskQueue, &uvTask->recvTaskQueue);
+  uv_mutex_unlock(&udfc->gUdfTaskQueueMutex);
+  uv_async_send(&udfc->gUdfLoopTaskAync);
 
   uv_sem_wait(&uvTask->taskSem);
   uv_sem_destroy(&uvTask->taskSem);
@@ -832,7 +830,7 @@ int32_t startUvUdfTask(SClientUvTaskNode *uvTask) {
   switch (uvTask->type) {
     case UV_TASK_CONNECT: {
       uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t));
-      uv_pipe_init(&gUdfdLoop, pipe, 0);
+      uv_pipe_init(&uvTask->udfc->gUdfdLoop, pipe, 0);
       uvTask->pipe = pipe;
 
       SClientUvConn *conn = taosMemoryMalloc(sizeof(SClientUvConn));
@@ -873,142 +871,98 @@ int32_t startUvUdfTask(SClientUvTaskNode *uvTask) {
 }
 
 void udfClientAsyncCb(uv_async_t *async) {
+  SUdfdProxy *udfc = async->data;
   QUEUE wq;
 
-  uv_mutex_lock(&gUdfTaskQueueMutex);
-  QUEUE_MOVE(&gUdfTaskQueue, &wq);
-  uv_mutex_unlock(&gUdfTaskQueueMutex);
+  uv_mutex_lock(&udfc->gUdfTaskQueueMutex);
+  QUEUE_MOVE(&udfc->gUdfTaskQueue, &wq);
+  uv_mutex_unlock(&udfc->gUdfTaskQueueMutex);
 
   while (!QUEUE_EMPTY(&wq)) {
     QUEUE* h = QUEUE_HEAD(&wq);
     QUEUE_REMOVE(h);
     SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, recvTaskQueue);
     startUvUdfTask(task);
-    QUEUE_INSERT_TAIL(&gUvProcTaskQueue, &task->procTaskQueue);
+    QUEUE_INSERT_TAIL(&udfc->gUvProcTaskQueue, &task->procTaskQueue);
   }
 
 }
 
-void cleanUpUvTasks() {
+void cleanUpUvTasks(SUdfdProxy *udfc) {
   QUEUE wq;
 
-  uv_mutex_lock(&gUdfTaskQueueMutex);
-  QUEUE_MOVE(&gUdfTaskQueue, &wq);
-  uv_mutex_unlock(&gUdfTaskQueueMutex);
+  uv_mutex_lock(&udfc->gUdfTaskQueueMutex);
+  QUEUE_MOVE(&udfc->gUdfTaskQueue, &wq);
+  uv_mutex_unlock(&udfc->gUdfTaskQueueMutex);
 
   while (!QUEUE_EMPTY(&wq)) {
     QUEUE* h = QUEUE_HEAD(&wq);
     QUEUE_REMOVE(h);
     SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, recvTaskQueue);
-    if (gUdfcState == UDFC_STATE_STOPPING) {
+    if (udfc->gUdfcState == UDFC_STATE_STOPPING) {
       task->errCode = UDFC_CODE_STOPPING;
-    } else if (gUdfcState == UDFC_STATE_RESTARTING) {
-      task->errCode = UDFC_CODE_RESTARTING;
     }
     uv_sem_post(&task->taskSem);
   }
 
   // TODO: deal with tasks that are waiting result.
-  while (!QUEUE_EMPTY(&gUvProcTaskQueue)) {
-    QUEUE* h = QUEUE_HEAD(&gUvProcTaskQueue);
+  while (!QUEUE_EMPTY(&udfc->gUvProcTaskQueue)) {
+    QUEUE* h = QUEUE_HEAD(&udfc->gUvProcTaskQueue);
     QUEUE_REMOVE(h);
     SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, procTaskQueue);
-    if (gUdfcState == UDFC_STATE_STOPPING) {
+    if (udfc->gUdfcState == UDFC_STATE_STOPPING) {
       task->errCode = UDFC_CODE_STOPPING;
-    } else if (gUdfcState == UDFC_STATE_RESTARTING) {
-      task->errCode = UDFC_CODE_RESTARTING;
     }
     uv_sem_post(&task->taskSem);
   }
 }
 
 void udfStopAsyncCb(uv_async_t *async) {
-  cleanUpUvTasks();
-  if (gUdfcState == UDFC_STATE_STOPPING) {
-    uv_stop(&gUdfdLoop);
+  SUdfdProxy *udfc = async->data;
+  cleanUpUvTasks(udfc);
+  if (udfc->gUdfcState == UDFC_STATE_STOPPING) {
+    uv_stop(&udfc->gUdfdLoop);
   }
 }
 
-int32_t udfcSpawnUdfd();
-
-void onUdfdExit(uv_process_t *req, int64_t exit_status, int term_signal) {
-  //TODO: pipe close will be first received
-  debugPrint("Process exited with status %" PRId64 ", signal %d", exit_status, term_signal);
-  uv_close((uv_handle_t *) req, NULL);
-  //TODO: restart the udfd process
-  if (gUdfcState == UDFC_STATE_STOPPING) {
-    if (term_signal != SIGINT) {
-      //TODO: log error
-    }
-  }
-  if (gUdfcState == UDFC_STATE_READY) {
-    gUdfcState = UDFC_STATE_RESTARTING;
-    //TODO: asynchronous without blocking. how to do it
-    //cleanUpUvTasks();
-    udfcSpawnUdfd();
-  }
-}
-
-int32_t udfcSpawnUdfd() {
-  //TODO: path
-  uv_process_options_t options = {0};
-  static char path[256] = {0};
-  size_t cwdSize;
-  uv_cwd(path, &cwdSize);
-  strcat(path, "/udfd");
-  char* args[2] = {path, NULL};
-  options.args = args;
-  options.file = path;
-  options.exit_cb = onUdfdExit;
-  options.stdio_count = 3;
-  uv_stdio_container_t child_stdio[3];
-  child_stdio[0].flags = UV_IGNORE;
-  child_stdio[1].flags = UV_INHERIT_FD;
-  child_stdio[1].data.fd = 1;
-  child_stdio[2].flags = UV_INHERIT_FD;
-  child_stdio[2].data.fd = 2;
-  options.stdio = child_stdio;
-  //TODO spawn error
-  int err = uv_spawn(&gUdfdLoop, &gUdfdProcess, &options);
-  if (err != 0) {
-    debugPrint("can not spawn udfd. path: %s, error: %s", path, uv_strerror(err));
-  }
-  return err;
-}
-
 void constructUdfService(void *argsThread) {
-  uv_loop_init(&gUdfdLoop);
-
-  uv_async_init(&gUdfdLoop, &gUdfLoopTaskAync, udfClientAsyncCb);
-  uv_async_init(&gUdfdLoop, &gUdfLoopStopAsync, udfStopAsyncCb);
-  uv_mutex_init(&gUdfTaskQueueMutex);
-  QUEUE_INIT(&gUdfTaskQueue);
-  QUEUE_INIT(&gUvProcTaskQueue);
-  uv_barrier_wait(&gUdfInitBarrier);
+  SUdfdProxy *udfc = (SUdfdProxy*)argsThread;
+  uv_loop_init(&udfc->gUdfdLoop);
+
+  uv_async_init(&udfc->gUdfdLoop, &udfc->gUdfLoopTaskAync, udfClientAsyncCb);
+  udfc->gUdfLoopTaskAync.data = udfc;
+  uv_async_init(&udfc->gUdfdLoop, &udfc->gUdfLoopStopAsync, udfStopAsyncCb);
+  udfc->gUdfLoopStopAsync.data = udfc;
+  uv_mutex_init(&udfc->gUdfTaskQueueMutex);
+  QUEUE_INIT(&udfc->gUdfTaskQueue);
+  QUEUE_INIT(&udfc->gUvProcTaskQueue);
+  uv_barrier_wait(&udfc->gUdfInitBarrier);
   //TODO return value of uv_run
-  uv_run(&gUdfdLoop, UV_RUN_DEFAULT);
-  uv_loop_close(&gUdfdLoop);
+  uv_run(&udfc->gUdfdLoop, UV_RUN_DEFAULT);
+  uv_loop_close(&udfc->gUdfdLoop);
 }
 
-
-int32_t createUdfdProxy(int32_t dnodeId) {
-  gUdfcState = UDFC_STATE_STARTNG;
-  uv_barrier_init(&gUdfInitBarrier, 2);
-  uv_thread_create(&gUdfLoopThread, constructUdfService, 0);
-  uv_barrier_wait(&gUdfInitBarrier);  gUdfcState = UDFC_STATE_READY;
+int32_t udfcOpen(int32_t dnodeId, UdfcHandle *udfc) {
+  SUdfdProxy *proxy = taosMemoryCalloc(1, sizeof(SUdfdProxy));
+  proxy->dnodeId = dnodeId;
+  proxy->gUdfcState = UDFC_STATE_STARTNG;
+  uv_barrier_init(&proxy->gUdfInitBarrier, 2);
+  uv_thread_create(&proxy->gUdfLoopThread, constructUdfService, proxy);
+  uv_barrier_wait(&proxy->gUdfInitBarrier);
+  proxy->gUdfcState = UDFC_STATE_READY;
+  *udfc = proxy;
   return 0;
 }
 
-int32_t destroyUdfdProxy(int32_t dnodeId) {
-  gUdfcState = UDFC_STATE_STOPPING;
-  uv_barrier_destroy(&gUdfInitBarrier);
-//  if (gUdfcState == UDFC_STATE_STOPPING) {
-//    uv_process_kill(&gUdfdProcess, SIGINT);
-//  }
-  uv_async_send(&gUdfLoopStopAsync);
-  uv_thread_join(&gUdfLoopThread);
-  uv_mutex_destroy(&gUdfTaskQueueMutex);
-  gUdfcState = UDFC_STATUS_FINAL;
+int32_t udfcClose(UdfcHandle udfcHandle) {
+  SUdfdProxy *udfc = udfcHandle;
+  udfc->gUdfcState = UDFC_STATE_STOPPING;
+  uv_async_send(&udfc->gUdfLoopStopAsync);
+  uv_thread_join(&udfc->gUdfLoopThread);
+  uv_mutex_destroy(&udfc->gUdfTaskQueueMutex);
+  uv_barrier_destroy(&udfc->gUdfInitBarrier);
+  udfc->gUdfcState = UDFC_STATUS_FINAL;
+  taosMemoryFree(udfc);
   return 0;
 }
 
@@ -1026,11 +980,12 @@ int32_t udfcRunUvTask(SClientUdfTask *task, int8_t uvTaskType) {
   return task->errCode;
 }
 
-int32_t setupUdf(char udfName[], SEpSet *epSet, UdfHandle *handle) {
+int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle *funcHandle) {
   debugPrint("%s", "client setup udf");
   SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask));
   task->errCode = 0;
   task->session = taosMemoryMalloc(sizeof(SUdfUvSession));
+  task->session->udfc = udfc;
   task->type = UDF_TASK_SETUP;
 
   SUdfSetupRequest *req = &task->_setup.req;
@@ -1046,13 +1001,13 @@ int32_t setupUdf(char udfName[], SEpSet *epSet, UdfHandle *handle) {
 
   SUdfSetupResponse *rsp = &task->_setup.rsp;
   task->session->severHandle = rsp->udfHandle;
-  *handle = task->session;
+  *funcHandle = task->session;
   int32_t err = task->errCode;
   taosMemoryFree(task);
   return err;
 }
 
-int32_t callUdf(UdfHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2,
+int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2,
                 SSDataBlock* output, SUdfInterBuf *newState) {
   debugPrint("%s", "client call udf");
 
@@ -1121,7 +1076,7 @@ int32_t callUdf(UdfHandle handle, int8_t callType, SSDataBlock *input, SUdfInter
 }
 
 //TODO: translate these calls to callUdf
-int32_t callUdfAggInit(UdfHandle handle, SUdfInterBuf *interBuf) {
+int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf) {
   int8_t callType = TSDB_UDF_CALL_AGG_INIT;
 
   int32_t err = callUdf(handle, callType, NULL, NULL, NULL, NULL, interBuf);
@@ -1131,7 +1086,7 @@ int32_t callUdfAggInit(UdfHandle handle, SUdfInterBuf *interBuf) {
 
 // input: block, state
 // output: interbuf,
-int32_t callUdfAggProcess(UdfHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState) {
+int32_t callUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState) {
   int8_t callType = TSDB_UDF_CALL_AGG_PROC;
   int32_t err = callUdf(handle, callType, block, state, NULL, NULL, newState);
   return err;
@@ -1139,7 +1094,7 @@ int32_t callUdfAggProcess(UdfHandle handle, SSDataBlock *block, SUdfInterBuf *st
 
 // input: interbuf1, interbuf2
 // output: resultBuf
-int32_t callUdfAggMerge(UdfHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf) {
+int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf) {
   int8_t callType = TSDB_UDF_CALL_AGG_MERGE;
   int32_t err = callUdf(handle, callType, NULL, interBuf1, interBuf2, NULL, resultBuf);
   return err;
@@ -1147,7 +1102,7 @@ int32_t callUdfAggMerge(UdfHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf
 
 // input: interBuf
 // output: resultData
-int32_t callUdfAggFinalize(UdfHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData) {
+int32_t callUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData) {
   int8_t callType = TSDB_UDF_CALL_AGG_PROC;
   int32_t err = callUdf(handle, callType, NULL, interBuf, NULL, NULL, resultData);
   return err;
@@ -1155,13 +1110,13 @@ int32_t callUdfAggFinalize(UdfHandle handle, SUdfInterBuf *interBuf, SUdfInterBu
 
 // input: block
 // output: resultData
-int32_t callUdfScalaProcess(UdfHandle handle, SSDataBlock *block, SSDataBlock *resultData) {
+int32_t callUdfScalaProcess(UdfcFuncHandle handle, SSDataBlock *block, SSDataBlock *resultData) {
   int8_t callType = TSDB_UDF_CALL_SCALA_PROC;
   int32_t err = callUdf(handle, callType, block, NULL, NULL, resultData, NULL);
   return err;
 }
 
-int32_t teardownUdf(UdfHandle handle) {
+int32_t teardownUdf(UdfcFuncHandle handle) {
   debugPrint("%s", "client teardown udf");
 
   SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask));

source/libs/function/src/udfd.c

@@ -27,7 +27,10 @@
 
 typedef struct SUdfdContext {
   uv_loop_t *loop;
+  uv_pipe_t ctrlPipe;
+  uv_signal_t intrSignal;
   char listenPipeName[UDF_LISTEN_PIPE_NAME_LEN];
+  uv_pipe_t listeningPipe;
   void      *clientRpc;
 
   uv_mutex_t udfsMutex;
@@ -76,9 +79,9 @@ typedef struct SUdf {
 
 // TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix
 // TODO: add private udf structure.
-typedef struct SUdfHandle {
+typedef struct SUdfcFuncHandle {
   SUdf *udf;
-} SUdfHandle;
+} SUdfcFuncHandle;
 
 int32_t udfdLoadUdf(char* udfName, SUdf* udf) {
     strcpy(udf->name, udfName);
@@ -143,7 +146,7 @@ void udfdProcessRequest(uv_work_t *req) {
         }
         uv_mutex_unlock(&udf->lock);
       }
-      SUdfHandle *handle = taosMemoryMalloc(sizeof(SUdfHandle));
+      SUdfcFuncHandle *handle = taosMemoryMalloc(sizeof(SUdfcFuncHandle));
       handle->udf = udf;
       // TODO: allocate private structure and call init function and set it to handle
       SUdfResponse rsp;
@@ -166,7 +169,7 @@ void udfdProcessRequest(uv_work_t *req) {
     case UDF_TASK_CALL: {
       SUdfCallRequest *call = &request.call;
       fnDebug("%"PRId64 "call request. call type %d, handle: %"PRIx64, request.seqNum, call->callType, call->udfHandle);
-      SUdfHandle      *handle = (SUdfHandle *)(call->udfHandle);
+      SUdfcFuncHandle      *handle = (SUdfcFuncHandle *)(call->udfHandle);
       SUdf            *udf = handle->udf;
 
       SUdfDataBlock input = {0};
@@ -204,7 +207,7 @@ void udfdProcessRequest(uv_work_t *req) {
     case UDF_TASK_TEARDOWN: {
       SUdfTeardownRequest *teardown = &request.teardown;
       fnInfo("teardown. %"PRId64"handle:%"PRIx64, request.seqNum, teardown->udfHandle)
-      SUdfHandle          *handle = (SUdfHandle *)(teardown->udfHandle);
+      SUdfcFuncHandle          *handle = (SUdfcFuncHandle *)(teardown->udfHandle);
       SUdf                *udf = handle->udf;
       bool unloadUdf = false;
       uv_mutex_lock(&global.udfsMutex);
@@ -380,10 +383,12 @@ void udfdOnNewConnection(uv_stream_t *server, int status) {
   }
 }
 
-void removeListeningPipe(int sig) {
+void udfdIntrSignalHandler(uv_signal_t *handle, int signum) {
+  fnInfo("udfd signal received: %d\n", signum);
   uv_fs_t req;
-  uv_fs_unlink(global.loop, &req, "udf.sock", NULL);
-  exit(0);
+  uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL);
+  uv_signal_stop(handle);
+  uv_stop(global.loop);
 }
 
 void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { return; }
@@ -492,37 +497,67 @@ static int32_t udfdInitLog() {
   return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, 0);
 }
 
+void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf) {
+  buf->base = taosMemoryMalloc(suggested_size);
+  buf->len = suggested_size;
+}
+
+void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf) {
+  if (nread < 0) {
+    fnError("udfd ctrl pipe read error. %s", uv_err_name(nread));
+    uv_close((uv_handle_t*)q, NULL);
+    uv_stop(global.loop);
+    return;
+  }
+  fnError("udfd ctrl pipe read %zu bytes", nread);
+  taosMemoryFree(buf->base);
+}
+
+static int32_t removeListeningPipe() {
+  uv_fs_t req;
+  int err = uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL);
+  uv_fs_req_cleanup(&req);
+  return err;
+}
+
 static int32_t udfdUvInit() {
   uv_loop_t* loop = taosMemoryMalloc(sizeof(uv_loop_t));
   if (loop) {
     uv_loop_init(loop);
   }
   global.loop = loop;
+
+  uv_pipe_init(global.loop, &global.ctrlPipe, 1);
+  uv_pipe_open(&global.ctrlPipe, 0);
+  uv_read_start((uv_stream_t*)&global.ctrlPipe, udfdCtrlAllocBufCb, udfdCtrlReadCb);
+
   char dnodeId[8] = {0};
   size_t dnodeIdSize;
-  uv_os_getenv("DNODE_ID", dnodeId, &dnodeIdSize);
+  int32_t err = uv_os_getenv("DNODE_ID", dnodeId, &dnodeIdSize);
+  if (err != 0) {
+    dnodeId[0] = '1';
+  }
   char listenPipeName[32] = {0};
   snprintf(listenPipeName, sizeof(listenPipeName), "%s%s", UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId);
   strcpy(global.listenPipeName, listenPipeName);
 
-  uv_fs_t req;
-  uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL);
+  removeListeningPipe();
 
-  uv_pipe_t server;
-  uv_pipe_init(global.loop, &server, 0);
+  uv_pipe_init(global.loop, &global.listeningPipe, 0);
 
-  signal(SIGINT, removeListeningPipe);
+  uv_signal_init(global.loop, &global.intrSignal);
+  uv_signal_start(&global.intrSignal, udfdIntrSignalHandler, SIGINT);
 
   int r;
   fnInfo("bind to pipe %s", global.listenPipeName);
-  if ((r = uv_pipe_bind(&server, listenPipeName))) {
+  if ((r = uv_pipe_bind(&global.listeningPipe, listenPipeName))) {
     fnError("Bind error %s", uv_err_name(r));
-    removeListeningPipe(0);
+    removeListeningPipe();
     return -1;
   }
-  if ((r = uv_listen((uv_stream_t *)&server, 128, udfdOnNewConnection))) {
+  if ((r = uv_listen((uv_stream_t *)&global.listeningPipe, 128, udfdOnNewConnection))) {
     fnError("Listen error %s", uv_err_name(r));
-    removeListeningPipe(0);
+    removeListeningPipe();
     return -2;
   }
   return 0;

source/libs/function/test/runUdf.c

@@ -8,7 +8,8 @@
 #include "tdatablock.h"
 
 int main(int argc, char *argv[]) {
-  createUdfdProxy(1);
+  UdfcHandle udfc;
+  udfcOpen(1, &udfc);
     uv_sleep(1000);
     char path[256] = {0};
     size_t cwdSize = 256;
@@ -20,9 +21,9 @@ int main(int argc, char *argv[]) {
     fprintf(stdout, "current working directory:%s\n", path);
     strcat(path, "/libudf1.so");
 
-    UdfHandle handle;
+    UdfcFuncHandle handle;
     SEpSet epSet;
-    setupUdf("udf1", &epSet, &handle);
+    setupUdf(udfc, "udf1", &epSet, &handle);
 
     SSDataBlock block = {0};
     SSDataBlock* pBlock = █
@@ -53,5 +54,5 @@ int main(int argc, char *argv[]) {
     }
     teardownUdf(handle);
 
-    destroyUdfdProxy(1);
+    udfcClose(udfc);
 }

source/libs/parser/src/parTranslater.c

@@ -585,7 +585,8 @@ static EDealRes haveAggFunction(SNode* pNode, void* pContext) {
 }
 
 static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) {
-  if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) {
+  SFmGetFuncInfoParam param = { .pCtg = pCxt->pParseCxt->pCatalog, .pRpc = pCxt->pParseCxt->pTransporter, .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet};
+  if (TSDB_CODE_SUCCESS != fmGetFuncInfo(&param, pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) {
     return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName);
   }
   pCxt->errCode = fmGetFuncResultType(pFunc, pCxt->msgBuf.buf, pCxt->msgBuf.len);
@@ -1918,7 +1919,8 @@ static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs
     return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_ROLLUP_OPTION);
   }
   SFunctionNode* pFunc = nodesListGetNode(pFuncs, 0);
-  if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) {
+  SFmGetFuncInfoParam param = { .pCtg = pCxt->pParseCxt->pCatalog, .pRpc = pCxt->pParseCxt->pTransporter, .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet};  
+  if (TSDB_CODE_SUCCESS != fmGetFuncInfo(&param, pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) {
     return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName);
   }
   return TSDB_CODE_SUCCESS;

source/libs/planner/src/planLogicCreater.c

@@ -85,13 +85,19 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) {
   return DEAL_RES_CONTINUE;
 }
 
-static int32_t rewriteExpr(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) {
+static int32_t rewriteExprForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) {
   nodesWalkExprs(pExprs, doNameExpr, NULL);
   SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs };
   nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt);
   return cxt.errCode;
 }
 
+static int32_t rewriteExprs(SNodeList* pExprs, SNodeList* pTarget) {
+  SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs };
+  nodesRewriteExprs(pTarget, doRewriteExpr, &cxt);
+  return cxt.errCode;
+}
+
 static int32_t pushLogicNode(SLogicPlanContext* pCxt, SLogicNode** pOldRoot, SLogicNode* pNewRoot) {
   if (NULL == pNewRoot->pChildren) {
     pNewRoot->pChildren = nodesMakeList();
@@ -433,10 +439,10 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect,
 
   // rewrite the expression in subsequent clauses
   if (TSDB_CODE_SUCCESS == code) {
-    code = rewriteExpr(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY);
+    code = rewriteExprForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = rewriteExpr(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY);
+    code = rewriteExprForSelect(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY);
   }
 
   if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving) {
@@ -472,7 +478,7 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm
   }
 
   if (TSDB_CODE_SUCCESS == code) {
-    code = rewriteExpr(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW);
+    code = rewriteExprForSelect(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW);
   }
 
   if (TSDB_CODE_SUCCESS == code) {
@@ -723,7 +729,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe
 
   // rewrite the expression in subsequent clauses
   if (TSDB_CODE_SUCCESS == code) {
-    code = rewriteExpr(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_DISTINCT);
+    code = rewriteExprForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_DISTINCT);
   }
 
   // set the output
@@ -850,6 +856,37 @@ static int32_t createSetOpProjectLogicNode(SLogicPlanContext* pCxt, SSetOperator
   return code;
 }
 
+static int32_t createSetOpAggLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetOperator, SLogicNode** pLogicNode) {
+  SAggLogicNode* pAgg = (SAggLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_AGG);
+  if (NULL == pAgg) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  int32_t code = TSDB_CODE_SUCCESS;
+  pAgg->pGroupKeys = nodesCloneList(pSetOperator->pProjectionList);
+  if (NULL == pAgg->pGroupKeys) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  // rewrite the expression in subsequent clauses
+  if (TSDB_CODE_SUCCESS == code) {
+    code = rewriteExprs(pAgg->pGroupKeys, pSetOperator->pOrderByList);
+  }
+
+  // set the output
+  if (TSDB_CODE_SUCCESS == code) {
+    code = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys, &pAgg->node.pTargets);
+  }
+
+  if (TSDB_CODE_SUCCESS == code) {
+    *pLogicNode = (SLogicNode*)pAgg;
+  } else {
+    nodesDestroyNode(pAgg);
+  }
+
+  return code;
+}
+
 static int32_t createSetOpLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetOperator, SLogicNode** pLogicNode) {
   SLogicNode* pSetOp = NULL;
   int32_t code = TSDB_CODE_SUCCESS;
@@ -857,6 +894,9 @@ static int32_t createSetOpLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetO
     case SET_OP_TYPE_UNION_ALL:
       code = createSetOpProjectLogicNode(pCxt, pSetOperator, &pSetOp);
       break;
+    case SET_OP_TYPE_UNION:
+      code = createSetOpAggLogicNode(pCxt, pSetOperator, &pSetOp);
+      break;
     default:
       code = -1;
       break;

source/libs/planner/src/planOptimizer.c

@@ -228,6 +228,8 @@ static int32_t cpdMergeConds(SNode** pDst, SNodeList** pSrc) {
     if (NULL == pLogicCond) {
       return TSDB_CODE_OUT_OF_MEMORY;
     }
+    pLogicCond->node.resType.type = TSDB_DATA_TYPE_BOOL;
+    pLogicCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;    
     pLogicCond->condType = LOGIC_COND_TYPE_AND;
     pLogicCond->pParameterList = *pSrc;
     *pDst = (SNode*)pLogicCond;

source/libs/planner/src/planSpliter.c

@@ -50,6 +50,11 @@ typedef struct SUaInfo {
   SLogicSubplan* pSubplan;
 } SUaInfo;
 
+typedef struct SUnInfo {
+  SAggLogicNode* pAgg;
+  SLogicSubplan* pSubplan;
+} SUnInfo;
+
 typedef bool (*FSplFindSplitNode)(SLogicSubplan* pSubplan, void* pInfo);
 
 static SLogicSubplan* splCreateScanSubplan(SSplitContext* pCxt, SScanLogicNode* pScan, int32_t flag) {
@@ -226,7 +231,6 @@ static SLogicSubplan* uaCreateSubplan(SSplitContext* pCxt, SLogicNode* pNode) {
   pSubplan->id.groupId = pCxt->groupId;
   pSubplan->subplanType = SUBPLAN_TYPE_SCAN;
   pSubplan->pNode = pNode;
-  // TSWAP(pSubplan->pVgroupList, ((SScanLogicNode*)pSubplan->pNode)->pVgroupList, SVgroupsInfo*);
   return pSubplan;
 }
 
@@ -244,24 +248,22 @@ static int32_t uaCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan
 
   pSubplan->subplanType = SUBPLAN_TYPE_MERGE;
 
-  return nodesListMakeAppend(&pProject->node.pChildren, (SNode*)pExchange);
-
-  // if (NULL == pProject->node.pParent) {
-  //   pSubplan->pNode = (SLogicNode*)pExchange;
-  //   nodesDestroyNode(pProject);
-  //   return TSDB_CODE_SUCCESS;
-  // }
-
-  // SNode* pNode;
-  // FOREACH(pNode, pProject->node.pParent->pChildren) {
-  //   if (nodesEqualNode(pNode, pProject)) {
-  //     REPLACE_NODE(pExchange);
-  //     nodesDestroyNode(pNode);
-  //     return TSDB_CODE_SUCCESS;
-  //   }
-  // }
-  // nodesDestroyNode(pExchange);
-  // return TSDB_CODE_FAILED;
+  if (NULL == pProject->node.pParent) {
+    pSubplan->pNode = (SLogicNode*)pExchange;
+    nodesDestroyNode(pProject);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SNode* pNode;
+  FOREACH(pNode, pProject->node.pParent->pChildren) {
+    if (nodesEqualNode(pNode, pProject)) {
+      REPLACE_NODE(pExchange);
+      nodesDestroyNode(pNode);
+      return TSDB_CODE_SUCCESS;
+    }
+  }
+  nodesDestroyNode(pExchange);
+  return TSDB_CODE_FAILED;
 }
 
 static int32_t uaSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) {
@@ -291,10 +293,78 @@ static int32_t uaSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) {
   return code;
 }
 
+static SLogicNode* unMatchByNode(SLogicNode* pNode) {
+  if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode) && LIST_LENGTH(pNode->pChildren) > 1) {
+    return pNode;
+  }
+  SNode* pChild;
+  FOREACH(pChild, pNode->pChildren) {
+    SLogicNode* pSplitNode = uaMatchByNode((SLogicNode*)pChild);
+    if (NULL != pSplitNode) {
+      return pSplitNode;
+    }
+  }
+  return NULL;
+}
+
+static int32_t unCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SAggLogicNode* pAgg) {
+  SExchangeLogicNode* pExchange = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_EXCHANGE);
+  if (NULL == pExchange) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  pExchange->srcGroupId = pCxt->groupId;
+  // pExchange->precision = pScan->pMeta->tableInfo.precision;
+  pExchange->node.pTargets = nodesCloneList(pAgg->node.pTargets);
+  if (NULL == pExchange->node.pTargets) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  pSubplan->subplanType = SUBPLAN_TYPE_MERGE;
+
+  return nodesListMakeAppend(&pAgg->node.pChildren, pExchange);
+}
+
+static bool unFindSplitNode(SLogicSubplan* pSubplan, SUnInfo* pInfo) {
+  SLogicNode* pSplitNode = unMatchByNode(pSubplan->pNode);
+  if (NULL != pSplitNode) {
+    pInfo->pAgg = (SAggLogicNode*)pSplitNode;
+    pInfo->pSubplan = pSubplan;
+  }
+  return NULL != pSplitNode;
+}
+
+static int32_t unSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) {
+  SUnInfo info = {0};
+  if (!splMatch(pCxt, pSubplan, 0, (FSplFindSplitNode)unFindSplitNode, &info)) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  int32_t code = TSDB_CODE_SUCCESS;
+
+  SNode* pChild = NULL;
+  FOREACH(pChild, info.pAgg->node.pChildren) {
+    code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, uaCreateSubplan(pCxt, (SLogicNode*)pChild));
+    if (TSDB_CODE_SUCCESS == code) {
+      REPLACE_NODE(NULL);
+    } else {
+      break;
+    }
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    nodesClearList(info.pAgg->node.pChildren);
+    info.pAgg->node.pChildren = NULL;
+    code = unCreateExchangeNode(pCxt, info.pSubplan, info.pAgg);
+  }
+  ++(pCxt->groupId);
+  pCxt->split = true;
+  return code;
+}
+
 static const SSplitRule splitRuleSet[] = {
   { .pName = "SuperTableScan", .splitFunc = stsSplit },
   { .pName = "ChildTableJoin", .splitFunc = ctjSplit },
   { .pName = "UnionAll",       .splitFunc = uaSplit  },
+  { .pName = "Union",          .splitFunc = unSplit  }
 };
 
 static const int32_t splitRuleNum = (sizeof(splitRuleSet) / sizeof(SSplitRule));

source/libs/planner/test/planSetOpTest.cpp

@@ -27,3 +27,9 @@ TEST_F(PlanSetOpTest, unionAll) {
 
   run("select c1, c2 from t1 where c1 > 10 union all select c1, c2 from t1 where c1 > 20");
 }
+
+TEST_F(PlanSetOpTest, union) {
+  useDb("root", "test");
+
+  run("select c1, c2 from t1 where c1 > 10 union select c1, c2 from t1 where c1 > 20");
+}

source/libs/planner/test/planTestUtil.cpp

@@ -62,7 +62,7 @@ public:
       doScaleOutLogicPlan(&cxt, pLogicSubplan, &pLogicPlan);
 
       SQueryPlan* pPlan = nullptr;
-      doCreatePhysiPlan(&cxt, pLogicPlan, &pPlan, NULL);
+      doCreatePhysiPlan(&cxt, pLogicPlan, &pPlan);
 
       if (g_isDump) {
         dump();
@@ -162,7 +162,8 @@ private:
     res_.scaledLogicPlan_ = toString((SNode*)(*pLogicPlan));
   }
 
-  void doCreatePhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList) {
+  void doCreatePhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan) {
+    SArray* pExecNodeList = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SQueryNodeAddr));
     DO_WITH_THROW(createPhysiPlan, pCxt, pLogicPlan, pPlan, pExecNodeList);
     res_.physiPlan_ = toString((SNode*)(*pPlan));
     SNode* pNode;

source/libs/qcom/src/querymsg.c

@@ -157,6 +157,28 @@ int32_t queryBuildGetIndexMsg(void *input, char **msg, int32_t msgSize, int32_t
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t queryBuildRetrieveFuncMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen) {
+  if (NULL == msg || NULL == msgLen) {
+    return TSDB_CODE_TSC_INVALID_INPUT;
+  }
+
+  SRetrieveFuncReq funcReq = {0};
+  funcReq.numOfFuncs = 1;
+  funcReq.ignoreCodeComment = true;
+  funcReq.pFuncNames = taosArrayInit(1, strlen(input) + 1);
+  taosArrayPush(funcReq.pFuncNames, input);
+
+  int32_t bufLen = tSerializeSRetrieveFuncReq(NULL, 0, &funcReq);
+  void   *pBuf = rpcMallocCont(bufLen);
+  tSerializeSRetrieveFuncReq(pBuf, bufLen, &funcReq);
+
+  taosArrayDestroy(funcReq.pFuncNames);
+
+  *msg = pBuf;
+  *msgLen = bufLen;
+
+  return TSDB_CODE_SUCCESS;
+}
 
 int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) {
   SUseDbOutput *pOut = output;
@@ -379,6 +401,31 @@ int32_t queryProcessGetIndexRsp(void *output, char *msg, int32_t msgSize) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t queryProcessRetrieveFuncRsp(void *output, char *msg, int32_t msgSize) {
+  SRetrieveFuncRsp out = {0};
+
+  if (NULL == output || NULL == msg || msgSize <= 0) {
+    return TSDB_CODE_TSC_INVALID_INPUT;
+  }
+
+  if (tDeserializeSRetrieveFuncRsp(msg, msgSize, &out) != 0) {
+    qError("tDeserializeSRetrieveFuncRsp failed, msgSize:%d", msgSize);
+    return TSDB_CODE_INVALID_MSG;
+  }
+
+  if (1 != out.numOfFuncs) {
+    qError("invalid func num returned, numOfFuncs:%d", out.numOfFuncs);
+    return TSDB_CODE_INVALID_MSG;
+  }
+
+  SFuncInfo * funcInfo = taosArrayGet(out.pFuncInfos, 0);
+
+  memcpy(output, funcInfo, sizeof(*funcInfo));
+  taosArrayDestroy(out.pFuncInfos);
+
+  return TSDB_CODE_SUCCESS;
+}
+
 void initQueryModuleMsgHandle() {
   queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryBuildTableMetaReqMsg;
   queryBuildMsg[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryBuildTableMetaReqMsg;
@@ -386,6 +433,7 @@ void initQueryModuleMsgHandle() {
   queryBuildMsg[TMSG_INDEX(TDMT_MND_QNODE_LIST)] = queryBuildQnodeListMsg;
   queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_DB_CFG)] = queryBuildGetDBCfgMsg;
   queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_INDEX)]  = queryBuildGetIndexMsg;
+  queryBuildMsg[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)]  = queryBuildRetrieveFuncMsg;
 
   queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryProcessTableMetaRsp;
   queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryProcessTableMetaRsp;
@@ -393,6 +441,7 @@ void initQueryModuleMsgHandle() {
   queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_QNODE_LIST)] = queryProcessQnodeListRsp;
   queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_DB_CFG)] = queryProcessGetDbCfgRsp;
   queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_INDEX)]  = queryProcessGetIndexRsp;
+  queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)]  = queryProcessRetrieveFuncRsp;
 }
 
 #pragma GCC diagnostic pop

source/libs/transport/inc/transportInt.h

@@ -63,13 +63,14 @@ typedef struct {
 
   void (*cfp)(void* parent, SRpcMsg*, SEpSet*);
   int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey);
+  int (*retry)(void* parent, SRpcMsg*, SEpSet*);
 
-  int32_t         refCount;
-  void*           parent;
-  void*           idPool;     // handle to ID pool
-  void*           tmrCtrl;    // handle to timer
-  SHashObj*       hash;       // handle returned by hash utility
-  void*           tcphandle;  // returned handle from TCP initialization
+  int32_t       refCount;
+  void*         parent;
+  void*         idPool;     // handle to ID pool
+  void*         tmrCtrl;    // handle to timer
+  SHashObj*     hash;       // handle returned by hash utility
+  void*         tcphandle;  // returned handle from TCP initialization
   TdThreadMutex mutex;
 } SRpcInfo;
 

source/libs/transport/src/trans.c

@@ -39,6 +39,7 @@ void* rpcOpen(const SRpcInit* pInit) {
   // register callback handle
   pRpc->cfp = pInit->cfp;
   pRpc->afp = pInit->afp;
+  pRpc->retry = pInit->rfp;
 
   if (pInit->connType == TAOS_CONN_SERVER) {
     pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads;
@@ -100,11 +101,10 @@ void rpcSendRedirectRsp(void* thandle, const SEpSet* pEpSet) {
   SRpcMsg rpcMsg;
   memset(&rpcMsg, 0, sizeof(rpcMsg));
 
-  rpcMsg.contLen = sizeof(SEpSet);
-  rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
-  if (rpcMsg.pCont == NULL) return;
-
-  memcpy(rpcMsg.pCont, pEpSet, sizeof(SEpSet));
+  SMEpSet msg = {.epSet = *pEpSet};
+  int32_t len = tSerializeSMEpSet(NULL, 0, &msg);
+  rpcMsg.pCont = rpcMallocCont(len);
+  tSerializeSMEpSet(rpcMsg.pCont, len, &msg);
 
   rpcMsg.code = TSDB_CODE_RPC_REDIRECT;
   rpcMsg.handle = thandle;

source/libs/transport/src/transCli.c

@@ -31,12 +31,8 @@ typedef struct SCliConn {
   int          hThrdIdx;
   STransCtx    ctx;
 
-  bool       broken;   // link broken or not
-  ConnStatus status;   //
-  int        release;  // 1: release
-  // spi configure
-  char spi;
-  char secured;
+  bool       broken;  // link broken or not
+  ConnStatus status;  //
 
   char*    ip;
   uint32_t port;
@@ -44,7 +40,6 @@ typedef struct SCliConn {
   // debug and log info
   struct sockaddr_in addr;
   struct sockaddr_in locaddr;
-
 } SCliConn;
 
 typedef struct SCliMsg {
@@ -102,6 +97,8 @@ static void cliSendCb(uv_write_t* req, int status);
 static void cliConnCb(uv_connect_t* req, int status);
 static void cliAsyncCb(uv_async_t* handle);
 
+static void cliAppCb(SCliConn* pConn, STransMsg* pMsg);
+
 static SCliConn* cliCreateConn(SCliThrdObj* thrd);
 static void      cliDestroyConn(SCliConn* pConn, bool clear /*clear tcp handle or not*/);
 static void      cliDestroy(uv_handle_t* handle);
@@ -303,8 +300,6 @@ void cliHandleResp(SCliConn* conn) {
          TMSG_INFO(pHead->msgType), taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port),
          taosInetNtoa(conn->locaddr.sin_addr), ntohs(conn->locaddr.sin_port), transMsg.contLen);
 
-  conn->secured = pHead->secured;
-
   if (pCtx == NULL && CONN_NO_PERSIST_BY_APP(conn)) {
     tTrace("except, server continue send while cli ignore it");
     // transUnrefCliHandle(conn);
@@ -318,7 +313,8 @@ void cliHandleResp(SCliConn* conn) {
 
   if (pCtx == NULL || pCtx->pSem == NULL) {
     tTrace("%s cli conn %p handle resp", pTransInst->label, conn);
-    (pTransInst->cfp)(pTransInst->parent, &transMsg, NULL);
+    cliAppCb(conn, &transMsg);
+    //(pTransInst->cfp)(pTransInst->parent, &transMsg, NULL);
   } else {
     tTrace("%s cli conn(sync) %p handle resp", pTransInst->label, conn);
     memcpy((char*)pCtx->pRsp, (char*)&transMsg, sizeof(transMsg));
@@ -384,7 +380,8 @@ void cliHandleExcept(SCliConn* pConn) {
         once = true;
         continue;
       }
-      (pTransInst->cfp)(pTransInst->parent, &transMsg, NULL);
+      cliAppCb(pConn, &transMsg);
+      //(pTransInst->cfp)(pTransInst->parent, &transMsg, NULL);
     } else {
       tTrace("%s cli conn(sync) %p handle except", pTransInst->label, pConn);
       memcpy((char*)(pCtx->pRsp), (char*)(&transMsg), sizeof(transMsg));
@@ -884,6 +881,18 @@ int cliRBChoseIdx(STrans* pTransInst) {
   }
   return index % pTransInst->numOfThreads;
 }
+void cliAppCb(SCliConn* pConn, STransMsg* transMsg) {
+  SCliThrdObj* pThrd = pConn->hostThrd;
+  STrans*      pTransInst = pThrd->pTransInst;
+
+  if (transMsg->code == TSDB_CODE_RPC_REDIRECT && pTransInst->retry != NULL) {
+    SMEpSet emsg = {0};
+    tDeserializeSMEpSet(transMsg->pCont, transMsg->contLen, &emsg);
+    pTransInst->retry(pTransInst, transMsg, &(emsg.epSet));
+  } else {
+    pTransInst->cfp(pTransInst->parent, transMsg, NULL);
+  }
+}
 
 void transCloseClient(void* arg) {
   SCliObj* cli = arg;

source/libs/transport/src/transSrv.c

@@ -30,7 +30,6 @@ typedef struct SSrvConn {
   uv_timer_t pTimer;
 
   queue       queue;
-  int         ref;
   int         persist;  // persist connection or not
   SConnBuffer readBuf;  // read buf,
   int         inType;
@@ -692,8 +691,6 @@ static void uvDestroyConn(uv_handle_t* handle) {
   if (thrd->quit && QUEUE_IS_EMPTY(&thrd->conn)) {
     tTrace("work thread quit");
     uv_walk(thrd->loop, uvWalkCb, NULL);
-    // uv_loop_close(thrd->loop);
-    // uv_stop(thrd->loop);
   }
 }
 
@@ -756,8 +753,6 @@ void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd) {
   thrd->quit = true;
   if (QUEUE_IS_EMPTY(&thrd->conn)) {
     uv_walk(thrd->loop, uvWalkCb, NULL);
-    // uv_loop_close(thrd->loop);
-    // uv_stop(thrd->loop);
   } else {
     destroyAllConn(thrd);
   }
@@ -851,10 +846,8 @@ void transRefSrvHandle(void* handle) {
   if (handle == NULL) {
     return;
   }
-  SSrvConn* conn = handle;
-
   int ref = T_REF_INC((SSrvConn*)handle);
-  UNUSED(ref);
+  tDebug("server conn %p ref count: %d", handle, ref);
 }
 
 void transUnrefSrvHandle(void* handle) {

source/os/src/osEnv.c

@@ -37,6 +37,7 @@ int64_t         tsOpenMax = 0;
 int64_t         tsStreamMax = 0;
 float           tsNumOfCores = 0;
 int64_t         tsTotalMemoryKB = 0;
+char*           tsProcPath = NULL;
 
 void osDefaultInit() {
   taosSeedRand(taosSafeRand());

source/os/src/osFile.c

@@ -380,7 +380,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
 #if FILE_WITH_LOCK
   taosThreadRwlockWrlock(&(pFile->rwlock));
 #endif
-  /*assert(pFile->fd >= 0); // Please check if you have closed the file.*/
+  assert(pFile->fd >= 0);  // Please check if you have closed the file.
 
   int64_t nleft = count;
   int64_t nwritten = 0;

source/os/src/osProc.c

@@ -17,7 +17,7 @@
 #define _DEFAULT_SOURCE
 #include "os.h"
 
-static char *tsProcPath = NULL;
+char *tsProcPath = NULL;
 
 int32_t taosNewProc(char **args) {
   int32_t pid = fork();

source/util/src/talgo.c

@@ -229,22 +229,21 @@ void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size
 }
 
 void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar,
-                    __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot) {
+                    __ext_compar_fn_t compar, char* buf, bool maxroot) {
   int32_t parent;
   int32_t child;
-  char   *buf;
+
+  char* tmp = NULL;
+  if (buf == NULL) {
+    tmp = taosMemoryMalloc(size);
+  } else {
+    tmp = buf;
+  }
 
   if (base && size > 0 && compar) {
     parent = start;
     child = 2 * parent + 1;
 
-    if (swap == NULL) {
-      buf = taosMemoryCalloc(1, size);
-      if (buf == NULL) {
-        return;
-      }
-    }
-
     if (maxroot) {
       while (child <= end) {
         if (child + 1 <= end &&
@@ -256,11 +255,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const
           break;
         }
 
-        if (swap == NULL) {
-          doswap(elePtrAt(base, size, parent), elePtrAt(base, size, child), size, buf);
-        } else {
-          (*swap)(elePtrAt(base, size, parent), elePtrAt(base, size, child), parswap);
-        }
+        doswap(elePtrAt(base, size, parent), elePtrAt(base, size, child), size, tmp);
 
         parent = child;
         child = 2 * parent + 1;
@@ -276,33 +271,35 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const
           break;
         }
 
-        if (swap == NULL) {
-          doswap(elePtrAt(base, size, parent), elePtrAt(base, size, child), size, buf);
-        } else {
-          (*swap)(elePtrAt(base, size, parent), elePtrAt(base, size, child), parswap);
-        }
+        doswap(elePtrAt(base, size, parent), elePtrAt(base, size, child), size, tmp);
 
         parent = child;
         child = 2 * parent + 1;
       }
     }
+  }
 
-    if (swap == NULL) {
-      taosMemoryFreeClear(buf);
-    }
+  if (buf == NULL) {
+    taosMemoryFree(tmp);
   }
 }
 
 void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar,
-                  const void *parswap, __ext_swap_fn_t swap, bool maxroot) {
+                  bool maxroot) {
   int32_t i;
 
+  char* buf = taosMemoryCalloc(1, size);
+  if (buf == NULL) {
+    return;
+  }
+
   if (base && size > 0) {
     for (i = len / 2 - 1; i >= 0; i--) {
-      taosheapadjust(base, size, i, len - 1, parcompar, compar, parswap, swap, maxroot);
+      taosheapadjust(base, size, i, len - 1, parcompar, compar, buf, maxroot);
     }
   }
 
+  taosMemoryFree(buf);
   /*
     char *buf = taosMemoryCalloc(1, size);
 

source/util/src/tlog.c

@@ -222,7 +222,7 @@ static void *taosThreadToOpenNewFile(void *param) {
   tsLogObj.logHandle->pFile = pFile;
   tsLogObj.lines = 0;
   tsLogObj.openInProgress = 0;
-  taosSsleep(3);
+  taosSsleep(10);
   taosCloseLogByFd(pOldFile);
 
   uInfo("   new log file:%d is opened", tsLogObj.flag);

tests/script/test.sh

@@ -131,8 +131,8 @@ if [ -n "$FILE_NAME" ]; then
       FLAG="-v"    
     fi 
   
-    echo valgrind --tool=memcheck --leak-check=full --show-reachable=no  --track-origins=yes --show-leak-kinds=all  -v  --workaround-gcc296-bugs=yes  --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG
-    valgrind --tool=memcheck --leak-check=full --show-reachable=no  --track-origins=yes --show-leak-kinds=all  -v  --workaround-gcc296-bugs=yes  --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG
+    echo valgrind --tool=memcheck --leak-check=full --show-reachable=no  --track-origins=yes --child-silent-after-fork=yes --show-leak-kinds=all  -v  --workaround-gcc296-bugs=yes  --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG
+    valgrind --tool=memcheck --leak-check=full --show-reachable=no  --track-origins=yes --child-silent-after-fork=yes --show-leak-kinds=all  -v  --workaround-gcc296-bugs=yes  --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG
   else
     if [[ $MULTIPROCESS -eq 1 ]];then
       echo "ExcuteCmd(multiprocess):" $PROGRAM -m -c $CFG_DIR -f $FILE_NAME  

tests/script/tsim/query/session.sim

@@ -279,7 +279,8 @@ endi
 
 print ================> syntax error check not active ================> reactive
 sql_error select * from dev_001 session(ts,1w)
-sql_error select count(*) from st session(ts,1w)
+print disable this temporarily, session can not be directly applied to super table.
+#sql_error select count(*) from st session(ts,1w)
 sql_error select count(*) from dev_001 group by tagtype session(ts,1w)
 sql_error sql select count(*) from dev_001 session(ts,1n)
 sql_error sql select count(*) from dev_001 session(ts,1y)

tests/script/tsim/tmq/basic1.sim

@@ -8,196 +8,245 @@
 #
 # notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
 #
-system sh/stop_dnodes.sh
 
-system sh/deploy.sh -n dnode1 -i 1
-system sh/exec.sh -n dnode1 -s start
-
-$loop_cnt = 0
-check_dnode_ready:
-	$loop_cnt = $loop_cnt + 1
-	sleep 200
-	if $loop_cnt == 10 then
-	  print ====> dnode not ready!
-		return -1
-	endi
-sql show dnodes
-print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
-if $data00 != 1 then
-  return -1
-endi
-if $data04 != ready then
-  goto check_dnode_ready
-endi
+run tsim/tmq/prepareBasicEnv.sim
+
+#---- global parameters start ----#
+$dbName    = db
+$vgroups    = 1
+$stbPrefix  = stb
+$ctbPrefix  = ctb
+$ntbPrefix  = ntb
+$stbNum     = 1
+$ctbNum     = 10
+$ntbNum     = 10
+$rowsPerCtb = 100
+$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
+#---- global parameters end ----#
+
+$pullDelay    = 3
+$ifcheckdata  = 1
+$showMsg      = 1
+$showRow      = 0
 
 sql connect
+sql use $dbName
 
-$dbNamme = d0
-print =============== create database , vgroup 4
-sql create database $dbNamme vgroups 4
-sql use $dbNamme
-
-print =============== create super table
-sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int)
-
-sql show stables
-if $rows != 1 then 
-  return -1
-endi
-
-print =============== create child table
-sql create table ct0 using stb tags(1000)
-sql create table ct1 using stb tags(2000)
-#sql create table ct3 using stb tags(3000)
+print == create topics from super table
+sql create topic topic_stb_column as select ts, c3 from stb
+sql create topic topic_stb_all as select ts, c1, c2, c3 from stb
+sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
 
-print =============== create normal table
-sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10))
+print == create topics from child table
+sql create topic topic_ctb_column as select ts, c3 from ctb0
+sql create topic topic_ctb_all as select * from ctb0
+sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0
 
-print =============== create multi topics. notes: now only support:
-print =============== 1. columns from stb; 2. * from ctb; 3. columns from ctb
-print =============== will support: * from stb; function from stb/ctb
+print == create topics from normal table
+sql create topic topic_ntb_column as select ts, c3 from ntb0
+sql create topic topic_ntb_all as select * from ntb0
+sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0
 
-sql create topic topic_stb_column as select ts, c1, c3 from stb
-#sql create topic topic_stb_all as select * from stb
-sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb
+#sql show topics
+#if $rows != 9 then 
+#  return -1
+#endi
 
-sql create topic topic_ctb_column as select ts, c1, c3 from ct0
-sql create topic topic_ctb_all as select * from ct0
-sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0
+$keyList = ' . group.id:cgrp1
+$keyList = $keyList . '
 
-sql create topic topic_ntb_column as select ts, c1, c3 from ntb
-sql create topic topic_ntb_all as select * from ntb
-sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb
+print ================ test consume from stb
+$loop_cnt = 0
+loop_consume_diff_topic_from_stb:
+if $loop_cnt == 0 then
+  print == scenario 1: topic_stb_column
+  $topicList = ' . topic_stb_column
+  $topicList = $topicList . '
+elif $loop_cnt == 1 then
+  print == scenario 2: topic_stb_all
+  $topicList = ' . topic_stb_all
+  $topicList = $topicList . '
+elif $loop_cnt == 2 then
+  print == scenario 3: topic_stb_function
+  $topicList = ' . topic_stb_function
+  $topicList = $topicList . '
+else
+  goto loop_consume_diff_topic_from_stb_end
+endi
 
-sql show tables
-if $rows != 3 then 
+$consumerId    = 0
+$totalMsgOfStb = $ctbNum * $rowsPerCtb
+#$expectmsgcnt  = $totalMsgOfStb + 1
+$expectmsgcnt  = 110
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+
+print == start consumer to pull msgs from stb
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start
+
+print == check consume result
+wait_consumer_end_from_stb:
+sql select * from consumeresult
+print ==> rows: $rows 
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+if $rows != 1 then
+  sleep 1000
+  goto wait_consumer_end_from_stb
+endi
+if $data[0][1] != $consumerId then
+  return -1
+endi
+if $data[0][2] != $expectmsgcnt then
   return -1
 endi
+if $data[0][3] != $expectmsgcnt then
+  return -1
+endi
+$loop_cnt = $loop_cnt + 1
+goto loop_consume_diff_topic_from_stb
+loop_consume_diff_topic_from_stb_end:
+
+#######################################################################################
+# clear consume info and consume result 
+#run tsim/tmq/clearConsume.sim
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
+$cdbName = cdb1
+sql create database $cdbName vgroups 1
+sleep 500
+sql use $cdbName
+
+print == create consume info table and consume result table
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
 
-print =============== insert data
-
-$tbPrefix = ct
-$tbNum = 2
-$rowNum = 10
-$tstart = 1640966400000  # 2022-01-01 00:00:00.000
-
-$i = 0
-while $i < $tbNum
-  $tb = $tbPrefix . $i
-
-  $x = 0
-  while $x < $rowNum
-    $c = $x / 10
-    $c = $c * 10
-    $c = $x - $c
-
-    $binary = ' . binary
-    $binary = $binary . $c
-    $binary = $binary . '
-
-    sql insert into $tb values ($tstart , $c , $x , $binary )
-    sql insert into ntb values ($tstart , $c , $x , $binary )
-    $tstart = $tstart + 1
-    $x = $x + 1
-  endw
-
-  $i = $i + 1
-  $tstart = 1640966400000
-endw
-
-#root@trd02 /home $ tmq_sim --help                                                                          
-# -c      Configuration directory, default is 
-# -d      The name of the database for cosumer, no default 
-# -t      The topic string for cosumer, no default 
-# -k      The key-value string for cosumer, no default 
-# -g      showMsgFlag, default is 0
-#        
-
-$totalMsgCnt = $rowNum * $tbNum
-print inserted totalMsgCnt: $totalMsgCnt
-
-# supported key:
-# group.id:<xxx>
-# enable.auto.commit:<true | false>
-# auto.offset.reset:<earliest | latest | none>
-# td.connect.ip:<fqdn | ipaddress>
-# td.connect.user:root
-# td.connect.pass:taosdata
-# td.connect.port:6030
-# td.connect.db:db
-
-system_content echo -n \$BUILD_DIR
-$tmq_sim = $system_content . /build/bin/tmq_sim
-system_content echo -n \$SIM_DIR
-$tsim_cfg = $system_content . /tsim/cfg
-
-print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2"
-system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2"
-print cmd result----> $system_content
-if $system_content != @{consume success: 20, 0}@ then
+sql show tables
+if $rows != 2 then 
   return -1
 endi
+#######################################################################################
 
-#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2"
-#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2"
-#print cmd result----> $system_content
-#if $system_content != @{consume success: 20, 0}@ then
-#  return -1
-#endi
 
-print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2"
-system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2"
-print cmd result----> $system_content
-if $system_content != @{consume success: 20, 0}@ then
-  print expect @{consume success: 20, 0}@, actual: $system_content
-  return -1
+print ================ test consume from ctb
+$loop_cnt = 0
+loop_consume_diff_topic_from_ctb:
+if $loop_cnt == 0 then
+  print == scenario 1: topic_ctb_column
+  $topicList = ' . topic_ctb_column
+  $topicList = $topicList . '
+elif $loop_cnt == 1 then
+  print == scenario 2: topic_ctb_all
+  $topicList = ' . topic_ctb_all
+  $topicList = $topicList . '
+elif $loop_cnt == 2 then
+  print == scenario 3: topic_ctb_function
+  $topicList = ' . topic_ctb_function
+  $topicList = $topicList . '
+else
+  goto loop_consume_diff_topic_from_ctb_end
 endi
 
-print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2"
-system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2"
-print cmd result----> $system_content
-if $system_content != @{consume success: 10, 0}@ then
+$consumerId    = 0
+$totalMsgOfCtb = $rowsPerCtb
+$expectmsgcnt  = $totalMsgOfCtb + 1
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+
+print == start consumer to pull msgs from stb
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+
+print == check consume result
+wait_consumer_end_from_ctb:
+sql select * from consumeresult
+print ==> rows: $rows 
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+if $rows != 1 then
+  sleep 1000
+  goto wait_consumer_end_from_ctb
+endi
+if $data[0][1] != $consumerId then
   return -1
 endi
-
-print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2"
-system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2"
-print cmd result----> $system_content
-if $system_content != @{consume success: 10, 0}@ then
+if $data[0][2] != $totalMsgOfCtb then
   return -1
 endi
-
-print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2"
-system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2"
-print cmd result----> $system_content
-if $system_content != @{consume success: 10, 0}@ then
+if $data[0][3] != $totalMsgOfCtb then
   return -1
 endi
+$loop_cnt = $loop_cnt + 1
+goto loop_consume_diff_topic_from_ctb
+loop_consume_diff_topic_from_ctb_end:
+
+#######################################################################################
+# clear consume info and consume result 
+#run tsim/tmq/clearConsume.sim
+# because drop table function no stable, so by create new db for consume info and result. Modify it later
+$cdbName = cdb2
+sql create database $cdbName vgroups 1
+sleep 500
+sql use $cdbName
+
+print == create consume info table and consume result table
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
 
-print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2"
-system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2"
-print cmd result----> $system_content
-if $system_content != @{consume success: 20, 0}@ then
+sql show tables
+if $rows != 2 then 
   return -1
 endi
+#######################################################################################
 
-print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2"
-system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2"
-print cmd result----> $system_content
-if $system_content != @{consume success: 20, 0}@ then
-  return -1
+
+print ================ test consume from ntb
+$loop_cnt = 0
+loop_consume_diff_topic_from_ntb:
+if $loop_cnt == 0 then
+  print == scenario 1: topic_ntb_column
+  $topicList = ' . topic_ntb_column
+  $topicList = $topicList . '
+elif $loop_cnt == 1 then
+  print == scenario 2: topic_ntb_all
+  $topicList = ' . topic_ntb_all
+  $topicList = $topicList . '
+elif $loop_cnt == 2 then
+  print == scenario 3: topic_ntb_function
+  $topicList = ' . topic_ntb_function
+  $topicList = $topicList . '
+else
+  goto loop_consume_diff_topic_from_ntb_end
 endi
 
-print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2"
-system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2"
-print cmd result----> $system_content
-if $system_content != @{consume success: 20, 0}@ then
+$consumerId    = 0
+$totalMsgOfNtb = $rowsPerCtb
+$expectmsgcnt  = $totalMsgOfNtb + 1
+sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata )
+
+print == start consumer to pull msgs from stb
+print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start
+system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start
+
+print == check consume result from ntb
+wait_consumer_end_from_ntb:
+sql select * from consumeresult
+print ==> rows: $rows 
+print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6]
+if $rows != 1 then
+  sleep 1000
+  goto wait_consumer_end_from_ntb
+endi
+if $data[0][1] != $consumerId then
   return -1
 endi
+if $data[0][2] != $totalMsgOfNtb then
+  return -1
+endi
+if $data[0][3] != $totalMsgOfNtb then
+  return -1
+endi
+$loop_cnt = $loop_cnt + 1
+goto loop_consume_diff_topic_from_ntb
+loop_consume_diff_topic_from_ntb_end:
 
-print =============== create database , vgroup 4
-$dbNamme = d1
-sql create database $dbNamme vgroups 4
-
+#------ not need stop consumer, because it exit after pull msg overthan expect msg
+#system tsim/tmq/consume.sh -s stop -x SIGINT
 
 system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/tmq/clearConsume.sim

+sql connect
+
+#---- global parameters start ----#
+$dbName    = db
+$vgroups    = 1
+$stbPrefix  = stb
+$ctbPrefix  = ctb
+$ntbPrefix  = ntb
+$stbNum     = 1
+$ctbNum     = 10
+$ntbNum     = 10
+$rowsPerCtb = 100
+$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
+#---- global parameters end ----#
+
+sql use $dbName
+
+print == create consume info table and consume result table
+sql drop table consumeinfo
+sql drop table consumeresult
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+
+sql show tables
+if $rows != 2 then 
+  return -1
+endi
+

tests/script/tsim/tmq/consume.sh

@@ -11,16 +11,25 @@ set +e
 # set default value for parameters
 EXEC_OPTON=start
 DB_NAME=db 
+CDB_NAME=db
 POLL_DELAY=5
 VALGRIND=0
 SIGNAL=SIGINT
+SHOW_MSG=0
+SHOW_ROW=0
 
-while getopts "d:s:v:y:x:" arg
+while getopts "d:s:v:y:x:g:r:w:" arg
 do
   case $arg in
     d)
       DB_NAME=$OPTARG
       ;;
+    g)
+      SHOW_MSG=$OPTARG
+      ;;
+    r)
+      SHOW_ROW=$OPTARG
+      ;;
     s)
       EXEC_OPTON=$OPTARG
       ;;
@@ -33,6 +42,9 @@ do
     x)
       SIGNAL=$OPTARG
       ;;
+    w)
+      CDB_NAME=$OPTARG
+      ;;
     ?)
       echo "unkown argument"
       ;;
@@ -80,11 +92,11 @@ echo "DB_NAME: $DB_NAME
 echo "------------------------------------------------------------------------"
 if [ "$EXEC_OPTON" = "start" ]; then 
   if [ $VALGRIND -eq 1 ]; then
-    echo nohup valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tmq_sim.log $PROGRAM -c $CFG_DIR -d $DB_NAME -y $POLL_DELAY > /dev/null 2>&1 &
-    nohup valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tmq_sim.log $PROGRAM -c $CFG_DIR -d $DB_NAME -y $POLL_DELAY > /dev/null 2>&1 &
+    echo nohup valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tmq_sim.log $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW > /dev/null 2>&1 &
+    nohup valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tmq_sim.log $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW > /dev/null 2>&1 &
   else
-    echo  "nohup $PROGRAM -c $CFG_DIR -d $DB_NAME -y $POLL_DELAY > /dev/null 2>&1 &"
-    nohup $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME > /dev/null 2>&1 &
+    echo  "nohup $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW -w $CDB_NAME > /dev/null 2>&1 &"
+    nohup $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW -w $CDB_NAME > /dev/null 2>&1 &
   fi
 else
   PID=`ps -ef|grep tmq_sim | grep -v grep | awk '{print $2}'`

tests/script/tsim/tmq/prepareBasicEnv.sim

+# stop all dnodes before start this case
+system sh/stop_dnodes.sh
+
+# deploy dnode 1
+system sh/deploy.sh -n dnode1 -i 1
+
+# add some config items for this case
+#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
+
+# start dnode 1
+system sh/exec.sh -n dnode1 -s start
+
+sql connect
+
+#---- global parameters start ----#
+$dbName    = db
+$vgroups    = 1
+$stbPrefix  = stb
+$ctbPrefix  = ctb
+$ntbPrefix  = ntb
+$stbNum     = 1
+$ctbNum     = 10
+$ntbNum     = 10
+$rowsPerCtb = 100
+$tstart     = 1640966400000  # 2022-01-01 00:00:00.000
+#---- global parameters end ----#
+
+print == create database $dbName vgroups $vgroups
+sql create database $dbName vgroups $vgroups
+
+#wait database ready
+$loop_cnt = 0
+check_db_ready:
+if $loop_cnt == 10 then
+  print ====> database not ready!
+	return -1
+endi
+sql show databases
+print ==> rows: $rows
+print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12]
+print     $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20]
+if $data(db)[20] != nostrict then
+  sleep 100
+  $loop_cnt = $loop_cnt + 1
+  goto check_db_ready  
+endi
+
+sql use $dbName
+
+print == create consume info table and consume result table
+sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)
+sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)
+
+sql show tables
+if $rows != 2 then 
+  return -1
+endi
+
+print == create super table
+sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int)
+sql show stables
+if $rows != 1 then 
+  return -1
+endi
+
+print == create child table, normal table and insert data
+$i = 0
+while $i < $ctbNum
+  $ctb = $ctbPrefix . $i
+  $ntb = $ntbPrefix . $i
+  sql create table $ctb using $stbPrefix tags( $i )
+  sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16))
+
+  $x = 0
+  while $x < $rowsPerCtb
+    $binary = ' . binary-
+    $binary = $binary . $i
+    $binary = $binary . '
+
+    sql insert into $ctb values ($tstart , $i , $x , $binary )
+    sql insert into $ntb values ($tstart , $i , $x , $binary )
+    $tstart = $tstart + 1
+    $x = $x + 1
+  endw
+
+  $i = $i + 1
+  $tstart = 1640966400000
+endw

tools/shell/src/shellEngine.c

@@ -224,27 +224,63 @@ int32_t shellRunCommand(TAOS *con, char *command) {
     }
   }
 
-  char quote = 0, *cmd = command;
+  bool esc = false;
+  char quote = 0, *cmd = command, *p = command;
   for (char c = *command++; c != 0; c = *command++) {
-    if (c == '\\' && (*command == '\'' || *command == '"' || *command == '`')) {
-      command ++;
+    if (esc) {
+      switch (c) {
+        case 'n':
+          c = '\n';
+          break;
+        case 'r':
+          c = '\r';
+          break;
+        case 't':
+          c = '\t';
+          break;
+        case 'G':
+          *p++ = '\\';
+          break;
+        case '\'':
+        case '"':
+          if (quote) {
+            *p++ = '\\';
+          }
+          break;
+      }
+      *p++ = c;
+      esc = false;
       continue;
     }
 
+    if (c == '\\') {
+      if (quote != 0 && (*command == '_' || *command == '\\')) {
+        // DO nothing
+      } else {
+        esc = true;
+        continue;
+      }
+    }
+
     if (quote == c) {
       quote = 0;
-    } else if (quote == 0 && (c == '\'' || c == '"' || c == '`')) {
+    } else if (quote == 0 && (c == '\'' || c == '"')) {
       quote = c;
-    } else if (c == ';' && quote == 0) {
-      c = *command;
-      *command = 0;
+    }
+
+    *p++ = c;
+    if (c == ';' && quote == 0) {
+      c = *p;
+      *p = 0;
       if (shellRunSingleCommand(con, cmd) < 0) {
         return -1;
       }
-      *command = c;
-      cmd = command;
+      *p = c;
+      p = cmd;
     }
   }
+
+  *p = 0;
   return shellRunSingleCommand(con, cmd);
 }
 
@@ -538,23 +574,19 @@ static void shellPrintNChar(const char *str, int length, int width) {
   while (pos < length) {
     TdWchar wc;
     int     bytes = taosMbToWchar(&wc, str + pos, MB_CUR_MAX);
-    if (bytes <= 0) {
+    if (bytes == 0) {
       break;
     }
-    if (pos + bytes > length) {
+    pos += bytes;
+    if (pos > length) {
       break;
     }
-    int w = 0;
+
 #ifdef WINDOWS
-    w = bytes;
+    int w = bytes;
 #else
-    if(*(str + pos) == '\t' || *(str + pos) == '\n' || *(str + pos) == '\r'){
-      w = bytes;
-    }else{
-      w = taosWcharWidth(wc);
-    }
+    int w = taosWcharWidth(wc);
 #endif
-    pos += bytes;
     if (w <= 0) {
       continue;
     }
@@ -648,7 +680,6 @@ static void printField(const char *val, TAOS_FIELD *field, int width, int32_t le
       break;
     case TSDB_DATA_TYPE_BINARY:
     case TSDB_DATA_TYPE_NCHAR:
-    case TSDB_DATA_TYPE_JSON:
       shellPrintNChar(val, length, width);
       break;
     case TSDB_DATA_TYPE_TIMESTAMP:
@@ -761,8 +792,7 @@ static int calcColWidth(TAOS_FIELD *field, int precision) {
         return TMAX(field->bytes, width);
       }
 
-    case TSDB_DATA_TYPE_NCHAR:
-    case TSDB_DATA_TYPE_JSON:{
+    case TSDB_DATA_TYPE_NCHAR: {
       int16_t bytes = field->bytes * TSDB_NCHAR_SIZE;
       if (bytes > tsMaxBinaryDisplayWidth) {
         return TMAX(tsMaxBinaryDisplayWidth, width);