Merge remote-tracking branch 'origin/3.0' into feature/qnode

include/client/taos.h

@@ -54,6 +54,7 @@ typedef void   TAOS_SUB;
 #define TSDB_DATA_TYPE_BLOB       18  // binary
 #define TSDB_DATA_TYPE_MEDIUMBLOB 19
 #define TSDB_DATA_TYPE_BINARY     TSDB_DATA_TYPE_VARCHAR  // string
+#define TSDB_DATA_TYPE_MAX        20
 
 typedef enum {
   TSDB_OPTION_LOCALE,
@@ -188,7 +189,7 @@ DLL_EXPORT bool        taos_is_null(TAOS_RES *res, int32_t row, int32_t col);
 DLL_EXPORT bool        taos_is_update_query(TAOS_RES *res);
 DLL_EXPORT int         taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows);
 DLL_EXPORT int         taos_validate_sql(TAOS *taos, const char *sql);
-DLL_EXPORT void taos_reset_current_db(TAOS *taos);
+DLL_EXPORT void        taos_reset_current_db(TAOS *taos);
 
 DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res);
 DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res);

include/common/taosdef.h

@@ -76,6 +76,13 @@ typedef enum {
   TSDB_SMA_TYPE_ROLLUP = 2,      // Rollup SMA
 } ETsdbSmaType;
 
+typedef enum {
+  TSDB_BSMA_TYPE_NONE = 0,  // no block-wise SMA
+  TSDB_BSMA_TYPE_I = 1,     // sum/min/max(default)
+} ETsdbBSmaType;
+
+#define TSDB_BSMA_TYPE_LATEST TSDB_BSMA_TYPE_I
+
 extern char *qtypeStr[];
 
 #define TSDB_PORT_HTTP      11

include/common/tcommon.h

@@ -53,12 +53,12 @@ typedef struct {
 
 typedef struct SColumnDataAgg {
   int16_t colId;
-  int64_t sum;
-  int64_t max;
-  int64_t min;
   int16_t maxIndex;
   int16_t minIndex;
   int16_t numOfNull;
+  int64_t sum;
+  int64_t max;
+  int64_t min;
 } SColumnDataAgg;
 
 typedef struct SDataBlockInfo {
@@ -193,20 +193,19 @@ typedef struct SColumn {
   uint8_t scale;
 } SColumn;
 
-typedef struct SLimit {
-  int64_t limit;
-  int64_t offset;
-} SLimit;
-
-typedef struct SOrder {
-  uint32_t order;
-  SColumn  col;
-} SOrder;
-
-typedef struct SGroupbyExpr {
-  SArray* columnInfo;  // SArray<SColIndex>, group by columns information
-  bool    groupbyTag;  // group by tag or column
-} SGroupbyExpr;
+typedef struct STableBlockDistInfo {
+  uint16_t  rowSize;
+  uint16_t  numOfFiles;
+  uint32_t  numOfTables;
+  uint64_t  totalSize;
+  uint64_t  totalRows;
+  int32_t   maxRows;
+  int32_t   minRows;
+  int32_t   firstSeekTimeUs;
+  uint32_t  numOfRowsInMemTable;
+  uint32_t  numOfSmallBlocks;
+  SArray   *dataBlockInfos;
+} STableBlockDistInfo;
 
 enum {
   FUNC_PARAM_TYPE_VALUE = 0x1,
@@ -241,15 +240,6 @@ typedef struct SExprInfo {
   struct tExprNode*     pExpr;
 } SExprInfo;
 
-typedef struct SStateWindow {
-  SColumn col;
-} SStateWindow;
-
-typedef struct SSessionWindow {
-  int64_t gap;  // gap between two session window(in microseconds)
-  SColumn col;
-} SSessionWindow;
-
 #define QUERY_ASC_FORWARD_STEP 1
 #define QUERY_DESC_FORWARD_STEP -1
 

include/common/tdatablock.h

@@ -113,6 +113,20 @@ static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uin
   pColumnInfoData->hasNull = true;
 }
 
+static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, uint32_t start, size_t nRows) {
+  if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
+    for(int32_t i = start; i < start + nRows; ++i) {
+      pColumnInfoData->varmeta.offset[i] = -1;  // it is a null value of VAR type.
+    }
+  } else {
+    for(int32_t i = start; i < start + nRows; ++i) {
+      colDataSetNull_f(pColumnInfoData->nullbitmap, i);
+    }
+  }
+
+  pColumnInfoData->hasNull = true;
+}
+
 static FORCE_INLINE int32_t colDataAppendInt8(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int8_t* v) {
   ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_TINYINT ||
          pColumnInfoData->info.type == TSDB_DATA_TYPE_UTINYINT || pColumnInfoData->info.type == TSDB_DATA_TYPE_BOOL);
@@ -179,12 +193,14 @@ size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock);
 int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo);
 int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
 
-int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows);
+int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows);
 int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
 void    blockDataCleanup(SSDataBlock* pDataBlock);
 size_t  blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize);
 void*   blockDataDestroy(SSDataBlock* pBlock);
 
+int32_t blockDataTrimFirstNRows(SSDataBlock *pBlock, size_t n);
+
 SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock);
 
 void blockDebugShowData(const SArray* dataBlocks);

include/common/tdataformat.h

@@ -70,11 +70,13 @@ typedef struct {
 #pragma pack(pop)
 
 #define colType(col)   ((col)->type)
+#define colSma(col)    ((col)->sma)
 #define colColId(col)  ((col)->colId)
 #define colBytes(col)  ((col)->bytes)
 #define colOffset(col) ((col)->offset)
 
 #define colSetType(col, t)   (colType(col) = (t))
+#define colSetSma(col, s)    (colSma(col) = (s))
 #define colSetColId(col, id) (colColId(col) = (id))
 #define colSetBytes(col, b)  (colBytes(col) = (b))
 #define colSetOffset(col, o) (colOffset(col) = (o))
@@ -133,13 +135,13 @@ typedef struct {
 #define TD_VTYPE_PARTS 4  // 8 bits / TD_VTYPE_BITS = 4
 #define TD_VTYPE_OPTR  3  // TD_VTYPE_PARTS - 1, utilize to get remainder
 
-#define TD_BITMAP_BYTES(cnt) (ceil((double)cnt / TD_VTYPE_PARTS))
-#define TD_BIT_TO_BYTES(cnt) (ceil((double)cnt / 8))
+#define TD_BITMAP_BYTES(cnt) (ceil((double)(cnt) / TD_VTYPE_PARTS))
+#define TD_BIT_TO_BYTES(cnt) (ceil((double)(cnt) / 8))
 
 int32_t   tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
 void      tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder);
 void      tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
-int32_t   tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col_bytes_t bytes);
+int32_t   tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col_id_t colId, col_bytes_t bytes);
 STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
 
 // ----------------- Semantic timestamp key definition
@@ -363,6 +365,7 @@ static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSche
 }
 #endif
 // ----------------- Data column structure
+// SDataCol arrangement: data => bitmap => dataOffset
 typedef struct SDataCol {
   int8_t          type;        // column type
   uint8_t         bitmap : 1;  // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM

include/common/tmsg.h

@@ -265,6 +265,20 @@ typedef struct SSchema {
   char     name[TSDB_COL_NAME_LEN];
 } SSchema;
 
+typedef struct {
+  int8_t   type;
+  int8_t   sma;  // ETsdbBSmaType and default is TSDB_BSMA_TYPE_I
+  col_id_t colId;
+  int32_t  bytes;
+  char     name[TSDB_COL_NAME_LEN];
+} SSchemaEx;
+
+#define SSCHMEA_TYPE(s) ((s)->type)
+#define SSCHMEA_SMA(s) ((s)->sma)
+#define SSCHMEA_COLID(s) ((s)->colId)
+#define SSCHMEA_BYTES(s) ((s)->bytes)
+#define SSCHMEA_NAME(s) ((s)->name)
+
 typedef struct {
   char    name[TSDB_TABLE_FNAME_LEN];
   int8_t  igExists;
@@ -524,6 +538,7 @@ typedef struct {
   int8_t  walLevel;
   int8_t  quorum;
   int8_t  cacheLastRow;
+  int8_t  replications;
 } SAlterDbReq;
 
 int32_t tSerializeSAlterDbReq(void* buf, int32_t bufLen, SAlterDbReq* pReq);
@@ -1396,11 +1411,10 @@ typedef struct {
 } SDDropTopicReq;
 
 typedef struct {
-  float    xFilesFactor;
-  int8_t   delayUnit;
-  int8_t   nFuncIds;
-  int32_t* pFuncIds;
-  int64_t  delay;
+  float      xFilesFactor;
+  int32_t    delay;
+  int8_t     nFuncIds;
+  func_id_t* pFuncIds;
 } SRSmaParam;
 
 typedef struct SVCreateTbReq {
@@ -1418,13 +1432,12 @@ typedef struct SVCreateTbReq {
   };
   union {
     struct {
-      tb_uid_t    suid;
-      uint32_t    nCols;
-      SSchema*    pSchema;
-      uint32_t    nTagCols;
-      SSchema*    pTagSchema;
-      col_id_t    nBSmaCols;
-      col_id_t*   pBSmaCols;
+      tb_uid_t   suid;
+      col_id_t   nCols;
+      col_id_t   nBSmaCols;
+      SSchemaEx* pSchema;
+      col_id_t   nTagCols;
+      SSchema*   pTagSchema;
       SRSmaParam* pRSmaParam;
     } stbCfg;
     struct {
@@ -1432,10 +1445,9 @@ typedef struct SVCreateTbReq {
       SKVRow   pTag;
     } ctbCfg;
     struct {
-      uint32_t    nCols;
-      SSchema*    pSchema;
-      col_id_t    nBSmaCols;
-      col_id_t*   pBSmaCols;
+      col_id_t   nCols;
+      col_id_t   nBSmaCols;
+      SSchemaEx* pSchema;
       SRSmaParam* pRSmaParam;
     } ntbCfg;
   };
@@ -1914,7 +1926,10 @@ int32_t tDecodeSMqCMCommitOffsetReq(SCoder* decoder, SMqCMCommitOffsetReq* pReq)
 
 typedef struct {
   uint32_t nCols;
-  SSchema* pSchema;
+  union {
+    SSchema*   pSchema;
+    SSchemaEx* pSchemaEx;
+  };
 } SSchemaWrapper;
 
 static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema) {

include/common/tmsgcb.h

@@ -50,7 +50,6 @@ typedef struct {
   PutToQueueFp            queueFps[QUEUE_MAX];
   GetQueueSizeFp          qsizeFp;
   SendReqFp               sendReqFp;
-  SendMnodeReqFp          sendMnodeReqFp;
   SendRspFp               sendRspFp;
   RegisterBrokenLinkArgFp registerBrokenLinkArgFp;
   ReleaseHandleFp         releaseHandleFp;
@@ -60,7 +59,6 @@ void    tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb);
 int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq);
 int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype);
 int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq);
-int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, SRpcMsg* pReq);
 void    tmsgSendRsp(const SRpcMsg* pRsp);
 void    tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg);
 void    tmsgReleaseHandle(void* handle, int8_t type);

include/common/trow.h

@@ -147,20 +147,20 @@ typedef struct {
 
 typedef struct {
   // basic info
-  int8_t  rowType;
-  int16_t sver;
-  STSRow *pBuf;
+  int8_t       rowType;
+  schema_ver_t sver;
+  STSRow      *pBuf;
 
   // extended info
-  int32_t flen;
-  int16_t nBoundCols;
-  int16_t nCols;
-  int16_t nBitmaps;
-  int16_t nBoundBitmaps;
-  int32_t offset;
-  void   *pBitmap;
-  void   *pOffset;
-  int32_t extendedRowSize;
+  int32_t  flen;
+  col_id_t nBoundCols;
+  col_id_t nCols;
+  col_id_t nBitmaps;
+  col_id_t nBoundBitmaps;
+  int32_t  offset;
+  void    *pBitmap;
+  void    *pOffset;
+  int32_t  extendedRowSize;
 } SRowBuilder;
 
 #define TD_ROW_HEAD_LEN (sizeof(STSRow))
@@ -448,9 +448,9 @@ static FORCE_INLINE int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t
   }
 #ifdef TD_SUPPORT_BITMAP
   // the primary TS key is stored separatedly
-  pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1);
+  pBuilder->nBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nCols - 1);
   if (nBoundCols > 0) {
-    pBuilder->nBoundBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1);
+    pBuilder->nBoundBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1);
   } else {
     pBuilder->nBoundBitmaps = 0;
   }

include/common/ttokendef.h

@@ -82,21 +82,21 @@
 #define TK_SINGLE_STABLE                   64
 #define TK_STREAM_MODE                     65
 #define TK_RETENTIONS                      66
-#define TK_TABLE                           67
-#define TK_NK_LP                           68
-#define TK_NK_RP                           69
-#define TK_STABLE                          70
-#define TK_ADD                             71
-#define TK_COLUMN                          72
-#define TK_MODIFY                          73
-#define TK_RENAME                          74
-#define TK_TAG                             75
-#define TK_SET                             76
-#define TK_NK_EQ                           77
-#define TK_USING                           78
-#define TK_TAGS                            79
-#define TK_NK_DOT                          80
-#define TK_NK_COMMA                        81
+#define TK_NK_COMMA                        67
+#define TK_TABLE                           68
+#define TK_NK_LP                           69
+#define TK_NK_RP                           70
+#define TK_STABLE                          71
+#define TK_ADD                             72
+#define TK_COLUMN                          73
+#define TK_MODIFY                          74
+#define TK_RENAME                          75
+#define TK_TAG                             76
+#define TK_SET                             77
+#define TK_NK_EQ                           78
+#define TK_USING                           79
+#define TK_TAGS                            80
+#define TK_NK_DOT                          81
 #define TK_COMMENT                         82
 #define TK_BOOL                            83
 #define TK_TINYINT                         84
@@ -131,69 +131,91 @@
 #define TK_FUNCTIONS                      113
 #define TK_INDEXES                        114
 #define TK_FROM                           115
-#define TK_LIKE                           116
-#define TK_INDEX                          117
-#define TK_FULLTEXT                       118
-#define TK_FUNCTION                       119
-#define TK_INTERVAL                       120
-#define TK_TOPIC                          121
-#define TK_AS                             122
-#define TK_DESC                           123
-#define TK_DESCRIBE                       124
-#define TK_RESET                          125
-#define TK_QUERY                          126
-#define TK_EXPLAIN                        127
-#define TK_ANALYZE                        128
-#define TK_VERBOSE                        129
-#define TK_NK_BOOL                        130
-#define TK_RATIO                          131
-#define TK_NULL                           132
-#define TK_NK_VARIABLE                    133
-#define TK_NK_UNDERLINE                   134
-#define TK_ROWTS                          135
-#define TK_TBNAME                         136
-#define TK_QSTARTTS                       137
-#define TK_QENDTS                         138
-#define TK_WSTARTTS                       139
-#define TK_WENDTS                         140
-#define TK_WDURATION                      141
-#define TK_BETWEEN                        142
-#define TK_IS                             143
-#define TK_NK_LT                          144
-#define TK_NK_GT                          145
-#define TK_NK_LE                          146
-#define TK_NK_GE                          147
-#define TK_NK_NE                          148
-#define TK_MATCH                          149
-#define TK_NMATCH                         150
-#define TK_IN                             151
-#define TK_JOIN                           152
-#define TK_INNER                          153
-#define TK_SELECT                         154
-#define TK_DISTINCT                       155
-#define TK_WHERE                          156
-#define TK_PARTITION                      157
-#define TK_BY                             158
-#define TK_SESSION                        159
-#define TK_STATE_WINDOW                   160
-#define TK_SLIDING                        161
-#define TK_FILL                           162
-#define TK_VALUE                          163
-#define TK_NONE                           164
-#define TK_PREV                           165
-#define TK_LINEAR                         166
-#define TK_NEXT                           167
-#define TK_GROUP                          168
-#define TK_HAVING                         169
-#define TK_ORDER                          170
-#define TK_SLIMIT                         171
-#define TK_SOFFSET                        172
-#define TK_LIMIT                          173
-#define TK_OFFSET                         174
-#define TK_ASC                            175
-#define TK_NULLS                          176
-#define TK_FIRST                          177
-#define TK_LAST                           178
+#define TK_ACCOUNTS                       116
+#define TK_APPS                           117
+#define TK_CONNECTIONS                    118
+#define TK_LICENCE                        119
+#define TK_QUERIES                        120
+#define TK_SCORES                         121
+#define TK_TOPICS                         122
+#define TK_VARIABLES                      123
+#define TK_LIKE                           124
+#define TK_INDEX                          125
+#define TK_FULLTEXT                       126
+#define TK_FUNCTION                       127
+#define TK_INTERVAL                       128
+#define TK_TOPIC                          129
+#define TK_AS                             130
+#define TK_DESC                           131
+#define TK_DESCRIBE                       132
+#define TK_RESET                          133
+#define TK_QUERY                          134
+#define TK_EXPLAIN                        135
+#define TK_ANALYZE                        136
+#define TK_VERBOSE                        137
+#define TK_NK_BOOL                        138
+#define TK_RATIO                          139
+#define TK_COMPACT                        140
+#define TK_VNODES                         141
+#define TK_IN                             142
+#define TK_OUTPUTTYPE                     143
+#define TK_AGGREGATE                      144
+#define TK_BUFSIZE                        145
+#define TK_STREAM                         146
+#define TK_INTO                           147
+#define TK_KILL                           148
+#define TK_CONNECTION                     149
+#define TK_MERGE                          150
+#define TK_VGROUP                         151
+#define TK_REDISTRIBUTE                   152
+#define TK_SPLIT                          153
+#define TK_SYNCDB                         154
+#define TK_NULL                           155
+#define TK_NK_VARIABLE                    156
+#define TK_NOW                            157
+#define TK_ROWTS                          158
+#define TK_TBNAME                         159
+#define TK_QSTARTTS                       160
+#define TK_QENDTS                         161
+#define TK_WSTARTTS                       162
+#define TK_WENDTS                         163
+#define TK_WDURATION                      164
+#define TK_BETWEEN                        165
+#define TK_IS                             166
+#define TK_NK_LT                          167
+#define TK_NK_GT                          168
+#define TK_NK_LE                          169
+#define TK_NK_GE                          170
+#define TK_NK_NE                          171
+#define TK_MATCH                          172
+#define TK_NMATCH                         173
+#define TK_JOIN                           174
+#define TK_INNER                          175
+#define TK_SELECT                         176
+#define TK_DISTINCT                       177
+#define TK_WHERE                          178
+#define TK_PARTITION                      179
+#define TK_BY                             180
+#define TK_SESSION                        181
+#define TK_STATE_WINDOW                   182
+#define TK_SLIDING                        183
+#define TK_FILL                           184
+#define TK_VALUE                          185
+#define TK_NONE                           186
+#define TK_PREV                           187
+#define TK_LINEAR                         188
+#define TK_NEXT                           189
+#define TK_GROUP                          190
+#define TK_HAVING                         191
+#define TK_ORDER                          192
+#define TK_SLIMIT                         193
+#define TK_SOFFSET                        194
+#define TK_LIMIT                          195
+#define TK_OFFSET                         196
+#define TK_ASC                            197
+#define TK_NULLS                          198
+#define TK_FIRST                          199
+#define TK_LAST                           200
 
 #define TK_NK_SPACE                       300
 #define TK_NK_COMMENT                     301
@@ -207,10 +229,8 @@
 #define TK_NK_COLON                       500
 #define TK_NK_BITNOT                      501
 #define TK_INSERT                         502
-#define TK_INTO                           503
-#define TK_NOW                            504
 #define TK_VALUES                         507
-#define TK_IMPORT                         507
+#define TK_IMPORT                         509
 #define TK_NK_SEMI                        508
 
 #define TK_NK_NIL                         65535

include/common/ttypes.h

@@ -31,6 +31,7 @@ typedef int16_t  col_id_t;
 typedef int8_t   col_type_t;
 typedef int32_t  col_bytes_t;
 typedef uint16_t schema_ver_t;
+typedef int32_t  func_id_t;
 
 #pragma pack(push, 1)
 typedef struct {

include/dnode/mnode/mnode.h

@@ -29,8 +29,7 @@ extern "C" {
 typedef struct SMnode SMnode;
 
 typedef struct {
-  int32_t  dnodeId;
-  int64_t  clusterId;
+  bool     deploy;
   int8_t   replica;
   int8_t   selfIndex;
   SReplica replicas[TSDB_MAX_REPLICA];

include/libs/function/function.h

@@ -52,7 +52,12 @@ typedef struct SFuncExecFuncs {
   FExecFinalize finalize;
 } SFuncExecFuncs;
 
-#define MAX_INTERVAL_TIME_WINDOW 1000000  // maximum allowed time windows in final results
+typedef struct SFileBlockInfo {
+  int32_t numBlocksOfStep;
+} SFileBlockInfo;
+
+#define TSDB_BLOCK_DIST_STEP_ROWS 8
+#define MAX_INTERVAL_TIME_WINDOW  1000000  // maximum allowed time windows in final results
 
 #define FUNCTION_TYPE_SCALAR       1
 #define FUNCTION_TYPE_AGG          2
@@ -101,10 +106,6 @@ typedef struct SFuncExecFuncs {
 #define FUNCTION_DERIVATIVE   32
 #define FUNCTION_BLKINFO      33
 
-#define FUNCTION_HISTOGRAM    34
-#define FUNCTION_HLL          35
-#define FUNCTION_MODE         36
-#define FUNCTION_SAMPLE       37
 
 #define FUNCTION_COV          38
 
@@ -183,11 +184,11 @@ typedef struct SqlFunctionCtx {
   int32_t          columnIndex;  // TODO remove it
   uint8_t          currentStage;  // record current running step, default: 0
   bool             isAggSet;
+  int64_t          startTs;       // timestamp range of current query when function is executed on a specific data block, TODO remove it
   /////////////////////////////////////////////////////////////////
   bool             stableQuery;
   int16_t          functionId;    // function id
   char *           pOutput;       // final result output buffer, point to sdata->data
-  int64_t          startTs;       // timestamp range of current query when function is executed on a specific data block
   int32_t          numOfParams;
   SVariant         param[4];      // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
   int64_t         *ptsList;       // corresponding timestamp array list

include/libs/function/functionMgt.h

@@ -72,10 +72,15 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_ATAN,
 
   // string function
-  FUNCTION_TYPE_CHAR_LENGTH = 1500,
+  FUNCTION_TYPE_LENGTH = 1500,
+  FUNCTION_TYPE_CHAR_LENGTH,
   FUNCTION_TYPE_CONCAT,
   FUNCTION_TYPE_CONCAT_WS,
-  FUNCTION_TYPE_LENGTH,
+  FUNCTION_TYPE_LOWER,
+  FUNCTION_TYPE_UPPER,
+  FUNCTION_TYPE_LTRIM,
+  FUNCTION_TYPE_RTRIM,
+  FUNCTION_TYPE_SUBSTR,
 
   // conversion function
   FUNCTION_TYPE_CAST = 2000,
@@ -127,6 +132,7 @@ bool fmIsStringFunc(int32_t funcId);
 bool fmIsDatetimeFunc(int32_t funcId);
 bool fmIsTimelineFunc(int32_t funcId);
 bool fmIsTimeorderFunc(int32_t funcId);
+bool fmIsPseudoColumnFunc(int32_t funcId);
 bool fmIsWindowPseudoColumnFunc(int32_t funcId);
 bool fmIsWindowClauseFunc(int32_t funcId);
 

include/libs/nodes/cmdnodes.h

@@ -38,7 +38,9 @@ typedef struct SDatabaseOptions {
   int32_t fsyncPeriod;
   int32_t maxRowsPerBlock;
   int32_t minRowsPerBlock;
-  int32_t keep;
+  int32_t keep0;
+  int32_t keep1;
+  int32_t keep2;
   int32_t precision;
   int32_t quorum;
   int32_t replica;
@@ -76,7 +78,9 @@ typedef struct SAlterDatabaseStmt {
 
 typedef struct STableOptions {
   ENodeType type;
-  int32_t keep;
+  int32_t keep0;
+  int32_t keep1;
+  int32_t keep2;
   int32_t ttl;
   char comments[TSDB_STB_COMMENT_LEN];
   SNodeList* pSma;
@@ -193,6 +197,12 @@ typedef struct SShowStmt {
   SNode* pTbNamePattern; // SValueNode
 } SShowStmt;
 
+typedef struct SShowCreatStmt {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+  char tableName[TSDB_TABLE_NAME_LEN];
+} SShowCreatStmt;
+
 typedef enum EIndexType {
   INDEX_TYPE_SMA = 1,
   INDEX_TYPE_FULLTEXT

include/libs/nodes/nodes.h

@@ -103,6 +103,15 @@ typedef enum ENodeType {
   QUERY_NODE_EXPLAIN_STMT,
   QUERY_NODE_DESCRIBE_STMT,
   QUERY_NODE_RESET_QUERY_CACHE_STMT,
+  QUERY_NODE_COMPACT_STMT,
+  QUERY_NODE_CREATE_FUNCTION_STMT,
+  QUERY_NODE_DROP_FUNCTION_STMT,
+  QUERY_NODE_CREATE_STREAM_STMT,
+  QUERY_NODE_DROP_STREAM_STMT,
+  QUERY_NODE_MERGE_VGROUP_STMT,
+  QUERY_NODE_REDISTRIBUTE_VGROUP_STMT,
+  QUERY_NODE_SPLIT_VGROUP_STMT,
+  QUERY_NODE_SYNCDB_STMT,
   QUERY_NODE_SHOW_DATABASES_STMT,
   QUERY_NODE_SHOW_TABLES_STMT,
   QUERY_NODE_SHOW_STABLES_STMT,
@@ -115,6 +124,18 @@ typedef enum ENodeType {
   QUERY_NODE_SHOW_FUNCTIONS_STMT,
   QUERY_NODE_SHOW_INDEXES_STMT,
   QUERY_NODE_SHOW_STREAMS_STMT,
+  QUERY_NODE_SHOW_APPS_STMT,
+  QUERY_NODE_SHOW_CONNECTIONS_STMT,
+  QUERY_NODE_SHOW_LICENCE_STMT,
+  QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
+  QUERY_NODE_SHOW_CREATE_TABLE_STMT,
+  QUERY_NODE_SHOW_CREATE_STABLE_STMT, 
+  QUERY_NODE_SHOW_QUERIES_STMT,
+  QUERY_NODE_SHOW_SCORES_STMT,
+  QUERY_NODE_SHOW_TOPICS_STMT,
+  QUERY_NODE_SHOW_VARIABLE_STMT,
+  QUERY_NODE_KILL_CONNECTION_STMT,
+  QUERY_NODE_KILL_QUERY_STMT,
 
   // logic plan node
   QUERY_NODE_LOGIC_PLAN_SCAN,
@@ -192,16 +213,16 @@ typedef enum EDealRes {
 } EDealRes;
 
 typedef EDealRes (*FNodeWalker)(SNode* pNode, void* pContext);
-void nodesWalkNode(SNodeptr pNode, FNodeWalker walker, void* pContext);
-void nodesWalkList(SNodeList* pList, FNodeWalker walker, void* pContext);
-void nodesWalkNodePostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext);
-void nodesWalkListPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext);
+void nodesWalkExpr(SNodeptr pNode, FNodeWalker walker, void* pContext);
+void nodesWalkExprs(SNodeList* pList, FNodeWalker walker, void* pContext);
+void nodesWalkExprPostOrder(SNodeptr pNode, FNodeWalker walker, void* pContext);
+void nodesWalkExprsPostOrder(SNodeList* pList, FNodeWalker walker, void* pContext);
 
 typedef EDealRes (*FNodeRewriter)(SNode** pNode, void* pContext);
-void nodesRewriteNode(SNode** pNode, FNodeRewriter rewriter, void* pContext);
-void nodesRewriteList(SNodeList* pList, FNodeRewriter rewriter, void* pContext);
-void nodesRewriteNodePostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext);
-void nodesRewriteListPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext);
+void nodesRewriteExpr(SNode** pNode, FNodeRewriter rewriter, void* pContext);
+void nodesRewriteExprs(SNodeList* pList, FNodeRewriter rewriter, void* pContext);
+void nodesRewriteExprPostOrder(SNode** pNode, FNodeRewriter rewriter, void* pContext);
+void nodesRewriteExprsPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* pContext);
 
 bool nodesEqualNode(const SNodeptr a, const SNodeptr b);
 

include/libs/nodes/querynodes.h

@@ -79,6 +79,7 @@ typedef struct SValueNode {
   char* literal;
   bool isDuration;
   bool translate;
+  bool genByCalc;
   union {
     bool b;
     int64_t i;
@@ -277,7 +278,6 @@ typedef struct SVnodeModifOpStmt {
   ENodeType   nodeType;
   ENodeType   sqlNodeType;
   SArray*     pDataBlocks;         // data block for each vgroup, SArray<SVgDataBlocks*>.
-  int8_t      schemaAttache;       // denote if submit block is built with table schema or not
   uint8_t     payloadType;         // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
   uint32_t    insertType;          // insert data from [file|sql statement| bound statement]
   const char* sql;                 // current sql statement position
@@ -306,6 +306,7 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNod
 
 bool nodesIsExprNode(const SNode* pNode);
 
+bool nodesIsUnaryOp(const SOperatorNode* pOp);
 bool nodesIsArithmeticOp(const SOperatorNode* pOp);
 bool nodesIsComparisonOp(const SOperatorNode* pOp);
 bool nodesIsJsonOp(const SOperatorNode* pOp);

include/libs/scalar/scalar.h

@@ -42,6 +42,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type);
 int32_t vectorGetConvertType(int32_t type1, int32_t type2);
 int32_t vectorConvertImpl(const SScalarParam* pIn, SScalarParam* pOut);
 
+/* Math functions */
 int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t logFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t powFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
@@ -58,6 +59,17 @@ int32_t ceilFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp
 int32_t floorFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t roundFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
+/* String functions */
+int32_t lengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t charLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t lowerFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t upperFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t ltrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+
 bool getTimePseudoFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 
 int32_t winStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

include/libs/transport/trpc.h

@@ -58,6 +58,9 @@ typedef struct {
   void    *pNode;
 } SNodeMsg;
 
+typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *);
+typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
+
 typedef struct SRpcInit {
   uint16_t localPort;     // local port
   char *   label;         // for debug purpose
@@ -74,10 +77,10 @@ typedef struct SRpcInit {
   char *ckey;     // ciphering key
 
   // call back to process incoming msg, code shall be ignored by server app
-  void (*cfp)(void *parent, SRpcMsg *, SEpSet *);
+  RpcCfp cfp;
 
   // call back to retrieve the client auth info, for server app only
-  int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
+  RpcAfp afp;;
 
   void *parent;
 } SRpcInit;

include/os/osMemory.h

@@ -32,13 +32,16 @@ extern "C" {
 void *taosMemoryMalloc(int32_t size);
 void *taosMemoryCalloc(int32_t num, int32_t size);
 void *taosMemoryRealloc(void *ptr, int32_t size);
+void *taosMemoryStrDup(void *ptr);
 void taosMemoryFree(const void *ptr);
 int32_t taosMemorySize(void *ptr);
 
 #define taosMemoryFreeClear(ptr) \
   do {                           \
-    taosMemoryFree(ptr);         \
-    (ptr)=NULL;                  \
+    if (ptr) {                   \
+      taosMemoryFree(ptr);       \
+      (ptr) = NULL;              \
+    }                            \
   } while (0)
 
 #ifdef __cplusplus

include/os/osProc.h

@@ -21,9 +21,11 @@ extern "C" {
 #endif
 
 int32_t taosNewProc(char **args);
+void    taosWaitProc(int32_t pid);
+void    taosKillProc(int32_t pid);
+bool    taosProcExist(int32_t pid);
 void    taosSetProcName(int32_t argc, char **argv, const char *name);
 void    taosSetProcPath(int32_t argc, char **argv);
-bool    taosProcExists(int32_t pid);
 
 #ifdef __cplusplus
 }

include/os/osShm.h

@@ -26,7 +26,7 @@ typedef struct {
   void*   ptr;
 } SShm;
 
-int32_t taosCreateShm(SShm *pShm, int32_t shmsize) ;
+int32_t taosCreateShm(SShm *pShm, int32_t key, int32_t shmsize) ;
 void    taosDropShm(SShm *pShm);
 int32_t taosAttachShm(SShm *pShm);
 

include/util/taoserror.h

@@ -479,6 +479,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INVALID_ENDPOINT          TAOS_DEF_ERROR_CODE(0, 0x2613)
 #define TSDB_CODE_PAR_EXPRIE_STATEMENT          TAOS_DEF_ERROR_CODE(0, 0x2614)
 #define TSDB_CODE_PAR_INTERVAL_VALUE_TOO_SMALL  TAOS_DEF_ERROR_CODE(0, 0x2615)
+#define TSDB_CODE_PAR_DB_NOT_SPECIFIED          TAOS_DEF_ERROR_CODE(0, 0x2616)
+#define TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME   TAOS_DEF_ERROR_CODE(0, 0x2617)
+#define TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR  TAOS_DEF_ERROR_CODE(0, 0x2618)
 
 #ifdef __cplusplus
 }

include/util/tcoding.h

@@ -57,6 +57,8 @@ static FORCE_INLINE void *taosDecodeFixedI8(const void *buf, int8_t *value) {
   return POINTER_SHIFT(buf, sizeof(*value));
 }
 
+static FORCE_INLINE void *taosSkipFixedLen(const void *buf, size_t len) { return POINTER_SHIFT(buf, len); }
+
 // ---- Fixed U16
 static FORCE_INLINE int32_t taosEncodeFixedU16(void **buf, uint16_t value) {
   if (buf != NULL) {

include/util/tdef.h

@@ -128,18 +128,20 @@ extern const int32_t TYPE_BYTES[15];
   } while (0)
 
 typedef enum EOperatorType {
-  // arithmetic operator
+  // binary arithmetic operator
   OP_TYPE_ADD = 1,
   OP_TYPE_SUB,
   OP_TYPE_MULTI,
   OP_TYPE_DIV,
   OP_TYPE_MOD,
+  // unary arithmetic operator
+  OP_TYPE_MINUS,
 
   // bit operator
   OP_TYPE_BIT_AND,
   OP_TYPE_BIT_OR,
 
-  // comparison operator
+  // binary comparison operator
   OP_TYPE_GREATER_THAN,
   OP_TYPE_GREATER_EQUAL,
   OP_TYPE_LOWER_THAN,
@@ -152,6 +154,7 @@ typedef enum EOperatorType {
   OP_TYPE_NOT_LIKE,
   OP_TYPE_MATCH,
   OP_TYPE_NMATCH,
+  // unary comparison operator
   OP_TYPE_IS_NULL,
   OP_TYPE_IS_NOT_NULL,
   OP_TYPE_IS_TRUE,
@@ -304,13 +307,13 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_TOTAL_BLOCKS     10000
 #define TSDB_DEFAULT_TOTAL_BLOCKS 6
 
-#define TSDB_MIN_DAYS_PER_FILE     1
-#define TSDB_MAX_DAYS_PER_FILE     3650
-#define TSDB_DEFAULT_DAYS_PER_FILE 10
+#define TSDB_MIN_DAYS_PER_FILE     60    // unit minute
+#define TSDB_MAX_DAYS_PER_FILE     (3650 * 1440)
+#define TSDB_DEFAULT_DAYS_PER_FILE (10 * 1440)
 
-#define TSDB_MIN_KEEP     1       // data in db to be reserved.
-#define TSDB_MAX_KEEP     365000  // data in db to be reserved.
-#define TSDB_DEFAULT_KEEP 3650    // ten years
+#define TSDB_MIN_KEEP     (1 * 1440)       // data in db to be reserved. unit minute
+#define TSDB_MAX_KEEP     (365000 * 1440) // data in db to be reserved.
+#define TSDB_DEFAULT_KEEP (3650 * 1440)    // ten years
 
 #define TSDB_MIN_MIN_ROW_FBLOCK     10
 #define TSDB_MAX_MIN_ROW_FBLOCK     1000
@@ -328,7 +331,7 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_FSYNC_PERIOD     180000  // millisecond
 #define TSDB_DEFAULT_FSYNC_PERIOD 3000    // three second
 
-#define TSDB_MIN_WAL_LEVEL     0
+#define TSDB_MIN_WAL_LEVEL     1
 #define TSDB_MAX_WAL_LEVEL     2
 #define TSDB_DEFAULT_WAL_LEVEL 1
 
@@ -477,9 +480,9 @@ enum {
   SND_WORKER_TYPE__UNIQUE,
 };
 
-#define MND_VGID -1
-#define QND_VGID 1
-#define VND_VGID 0
+#define MNODE_HANDLE -1
+#define QNODE_HANDLE 1
+#define DEFAULT_HANDLE 0
 
 #define MAX_NUM_STR_SIZE 40
 

include/util/tprocess.h

@@ -22,13 +22,12 @@
 extern "C" {
 #endif
 
-typedef enum { PROC_REQ, PROC_RSP, PROC_REG, PROC_RELEASE } ProcFuncType;
+typedef enum { PROC_QUEUE, PROC_REQ, PROC_RSP, PROC_REGIST, PROC_RELEASE } ProcFuncType;
 
-typedef struct SProcQueue SProcQueue;
-typedef struct SProcObj   SProcObj;
+typedef struct SProcObj SProcObj;
 typedef void *(*ProcMallocFp)(int32_t contLen);
 typedef void *(*ProcFreeFp)(void *pCont);
-typedef void *(*ProcConsumeFp)(void *pParent, void *pHead, int16_t headLen, void *pBody, int32_t bodyLen,
+typedef void *(*ProcConsumeFp)(void *parent, void *pHead, int16_t headLen, void *pBody, int32_t bodyLen,
                                ProcFuncType ftype);
 
 typedef struct {
@@ -43,7 +42,7 @@ typedef struct {
   ProcMallocFp  parentMallocBodyFp;
   ProcFreeFp    parentFreeBodyFp;
   SShm          shm;
-  void         *pParent;
+  void         *parent;
   const char   *name;
   bool          isChild;
 } SProcCfg;
@@ -51,10 +50,13 @@ typedef struct {
 SProcObj *taosProcInit(const SProcCfg *pCfg);
 void      taosProcCleanup(SProcObj *pProc);
 int32_t   taosProcRun(SProcObj *pProc);
-int32_t   taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
-                              ProcFuncType ftype);
-int32_t   taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
-                               ProcFuncType ftype);
+
+int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
+                            void *handle, ProcFuncType ftype);
+void    taosProcRemoveHandle(SProcObj *pProc, void *handle);
+void    taosProcCloseHandles(SProcObj *pProc, void (*HandleFp)(void *handle));
+void    taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
+                             ProcFuncType ftype);
 
 #ifdef __cplusplus
 }

source/client/inc/clientInt.h

@@ -136,7 +136,7 @@ typedef struct STscObj {
   uint32_t        connId;
   int32_t         connType;
   uint64_t        id;         // ref ID returned by taosAddRef
-  TdThreadMutex mutex;      // used to protect the operation on db
+  TdThreadMutex   mutex;      // used to protect the operation on db
   int32_t         numOfReqs;  // number of sqlObj bound to this connection
   SAppInstInfo*   pAppInfo;
 } STscObj;
@@ -152,7 +152,8 @@ typedef struct SResultColumn {
 typedef struct SReqResultInfo {
   const char*    pRspMsg;
   const char*    pData;
-  TAOS_FIELD*    fields;
+  TAOS_FIELD*    fields;     // todo, column names are not needed.
+  TAOS_FIELD*    userFields; // the fields info that return to user
   uint32_t       numOfCols;
   int32_t*       length;
   char**         convertBuf;
@@ -221,6 +222,7 @@ void  destroyRequest(SRequestObj* pRequest);
 
 char* getDbOfConnection(STscObj* pObj);
 void  setConnectionDB(STscObj* pTscObj, const char* db);
+void  resetConnectDB(STscObj* pTscObj);
 
 void taos_init_imp(void);
 int  taos_options_imp(TSDB_OPTION option, const char* str);

source/client/src/clientEnv.c

@@ -169,6 +169,7 @@ static void doFreeReqResultInfo(SReqResultInfo *pResInfo) {
   taosMemoryFreeClear(pResInfo->row);
   taosMemoryFreeClear(pResInfo->pCol);
   taosMemoryFreeClear(pResInfo->fields);
+  taosMemoryFreeClear(pResInfo->userFields);
 
   if (pResInfo->convertBuf != NULL) {
     for (int32_t i = 0; i < pResInfo->numOfCols; ++i) {

source/client/src/clientImpl.c

@@ -56,7 +56,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass,
   }
 
   char localDb[TSDB_DB_NAME_LEN] = {0};
-  if (db != NULL) {
+  if (db != NULL && strlen(db) > 0) {
     if (!validateDbName(db)) {
       terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH;
       return NULL;
@@ -164,6 +164,7 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery) {
     if ((*pQuery)->haveResultSet) {
       setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols);
     }
+
     TSWAP(pRequest->dbList, (*pQuery)->pDbList, SArray*);
     TSWAP(pRequest->tableList, (*pQuery)->pTableList, SArray*);
   }
@@ -228,12 +229,24 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t
   assert(pSchema != NULL && numOfCols > 0);
 
   pResInfo->numOfCols = numOfCols;
-  pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(pSchema[0]));
+  pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD));
+  pResInfo->userFields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD));
 
   for (int32_t i = 0; i < pResInfo->numOfCols; ++i) {
     pResInfo->fields[i].bytes = pSchema[i].bytes;
-    pResInfo->fields[i].type = pSchema[i].type;
+    pResInfo->fields[i].type  = pSchema[i].type;
+
+    pResInfo->userFields[i].bytes = pSchema[i].bytes;
+    pResInfo->userFields[i].type  = pSchema[i].type;
+
+    if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR) {
+      pResInfo->userFields[i].bytes -= VARSTR_HEADER_SIZE;
+    } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
+      pResInfo->userFields[i].bytes = (pResInfo->userFields[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
+    }
+
     tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name));
+    tstrncpy(pResInfo->userFields[i].name, pSchema[i].name, tListLen(pResInfo->userFields[i].name));
   }
 }
 
@@ -740,6 +753,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
     pStart += colLength[i];
   }
 
+  // convert UCS4-LE encoded character to native multi-bytes character in current data block.
   for (int32_t i = 0; i < numOfCols; ++i) {
     int32_t type = pResultInfo->fields[i].type;
     int32_t bytes = pResultInfo->fields[i].bytes;
@@ -766,6 +780,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
       }
 
       pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i];
+      pResultInfo->row[i] = pResultInfo->pCol[i].pData;
     }
   }
 
@@ -791,6 +806,16 @@ void setConnectionDB(STscObj* pTscObj, const char* db) {
   taosThreadMutexUnlock(&pTscObj->mutex);
 }
 
+void resetConnectDB(STscObj* pTscObj) {
+  if (pTscObj == NULL) {
+    return;
+  }
+
+  taosThreadMutexLock(&pTscObj->mutex);
+  pTscObj->db[0] = 0;
+  taosThreadMutexUnlock(&pTscObj->mutex);
+}
+
 int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) {
   assert(pResultInfo != NULL && pRsp != NULL);
 

source/client/src/clientMain.c

@@ -146,7 +146,7 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
   }
 
   SReqResultInfo *pResInfo = &(((SRequestObj *)res)->body.resInfo);
-  return pResInfo->fields;
+  return pResInfo->userFields;
 }
 
 TAOS_RES *taos_query(TAOS *taos, const char *sql) {
@@ -264,8 +264,17 @@ int *taos_fetch_lengths(TAOS_RES *res) {
 }
 
 TAOS_ROW *taos_result_block(TAOS_RES *res) {
-  // TODO
-  return NULL;
+  SRequestObj* pRequest = (SRequestObj*) res;
+  if (pRequest == NULL) {
+    terrno = TSDB_CODE_INVALID_PARA;
+    return NULL;
+  }
+
+  if (taos_is_update_query(res)) {
+    return NULL;
+  }
+
+  return &pRequest->body.resInfo.row;
 }
 
 // todo intergrate with tDataTypes
@@ -313,7 +322,9 @@ int taos_affected_rows(TAOS_RES *res) {
   return pResInfo->numOfRows;
 }
 
-int taos_result_precision(TAOS_RES *res) { return TSDB_TIME_PRECISION_MILLI; }
+int taos_result_precision(TAOS_RES *res) {
+  return TSDB_TIME_PRECISION_MILLI;
+}
 
 int taos_select_db(TAOS *taos, const char *db) {
   STscObj *pObj = (STscObj *)taos;
@@ -365,8 +376,7 @@ bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) {
 }
 
 bool taos_is_update_query(TAOS_RES *res) {
-  // TODO
-  return true;
+  return taos_num_fields(res) == 0;
 }
 
 int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
@@ -393,8 +403,11 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
 int taos_validate_sql(TAOS *taos, const char *sql) { return true; }
 
 void taos_reset_current_db(TAOS *taos) {
-  // TODO
-  return;
+  if (taos == NULL) {
+    return;
+  }
+
+  resetConnectDB(taos);
 }
 
 const char *taos_get_server_info(TAOS *taos) {

source/client/src/tmq.c

@@ -78,6 +78,8 @@ struct tmq_t {
   STscObj*       pTscObj;
   tmq_commit_cb* commit_cb;
   int32_t        nextTopicIdx;
+  int8_t         epStatus;
+  int32_t        epSkipCnt;
   int32_t        waitingRequest;
   int32_t        readyRequest;
   SArray*        clientTopics;  // SArray<SMqClientTopic>
@@ -106,6 +108,7 @@ typedef struct {
   // connection info
   int32_t vgId;
   int32_t vgStatus;
+  int64_t skipCnt;
   SEpSet  epSet;
 } SMqClientVg;
 
@@ -137,6 +140,7 @@ typedef struct {
   tmq_t*          tmq;
   SMqClientVg*    pVg;
   int32_t         epoch;
+  int32_t         vgId;
   tsem_t          rspSem;
   tmq_message_t** msg;
   int32_t         sync;
@@ -311,6 +315,8 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
   pTmq->epoch = 0;
   pTmq->waitingRequest = 0;
   pTmq->readyRequest = 0;
+  pTmq->epStatus = 0;
+  pTmq->epSkipCnt = 0;
   // set conf
   strcpy(pTmq->clientId, conf->clientId);
   strcpy(pTmq->groupId, conf->groupId);
@@ -346,6 +352,8 @@ tmq_t* tmq_consumer_new1(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
   pTmq->epoch = 0;
   pTmq->waitingRequest = 0;
   pTmq->readyRequest = 0;
+  pTmq->epStatus = 0;
+  pTmq->epSkipCnt = 0;
   // set conf
   strcpy(pTmq->clientId, conf->clientId);
   strcpy(pTmq->groupId, conf->groupId);
@@ -832,21 +840,21 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   SMqClientVg*    pVg = pParam->pVg;
   tmq_t*          tmq = pParam->tmq;
   if (code != 0) {
-    tscWarn("msg discard, code:%x", code);
-    goto WRITE_QUEUE_FAIL;
+    tscWarn("msg discard from vg %d, epoch %d, code:%x", pParam->vgId, pParam->epoch, code);
+    goto CREATE_MSG_FAIL;
   }
 
   int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch;
   int32_t tmqEpoch = atomic_load_32(&tmq->epoch);
   if (msgEpoch < tmqEpoch) {
     /*printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);*/
-    tsem_post(&tmq->rspSem);
-    tscWarn("discard rsp epoch %d, current epoch %d", msgEpoch, tmqEpoch);
+    /*tsem_post(&tmq->rspSem);*/
+    tscWarn("discard rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch);
     return 0;
   }
 
   if (msgEpoch != tmqEpoch) {
-    tscWarn("mismatch rsp epoch %d, current epoch %d", msgEpoch, tmqEpoch);
+    tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch);
   } else {
     atomic_sub_fetch_32(&tmq->waitingRequest, 1);
   }
@@ -873,7 +881,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   /*SMqConsumeRsp* pRsp = taosMemoryCalloc(1, sizeof(SMqConsumeRsp));*/
   tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t));
   if (pRsp == NULL) {
-    goto WRITE_QUEUE_FAIL;
+    goto CREATE_MSG_FAIL;
   }
   memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));
   tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->msg);
@@ -882,35 +890,37 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   // TODO: alloc mem
   /*pRsp->*/
   /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/
+#if 0
   if (pRsp->msg.numOfTopics == 0) {
     /*printf("no data\n");*/
     taosFreeQitem(pRsp);
-    goto WRITE_QUEUE_FAIL;
+    goto CREATE_MSG_FAIL;
   }
+#endif
 
-  tscError("tmq recv poll: vg %d, req offset %ld, rsp offset %ld", pParam->pVg->vgId, pRsp->msg.reqOffset,
+  tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pParam->pVg->vgId, pRsp->msg.reqOffset,
            pRsp->msg.rspOffset);
 
   pRsp->vg = pParam->pVg;
   taosWriteQitem(tmq->mqueue, pRsp);
   atomic_add_fetch_32(&tmq->readyRequest, 1);
-  tsem_post(&tmq->rspSem);
+  /*tsem_post(&tmq->rspSem);*/
   return 0;
 
-WRITE_QUEUE_FAIL:
+CREATE_MSG_FAIL:
   if (pParam->epoch == tmq->epoch) {
     atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
   }
-  tsem_post(&tmq->rspSem);
+  /*tsem_post(&tmq->rspSem);*/
   return code;
 }
 
 bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
   /*printf("call update ep %d\n", epoch);*/
-  /*printf("tmq update ep epoch %d to epoch %d\n", tmq->epoch, epoch);*/
   bool    set = false;
   int32_t topicNumGet = taosArrayGetSize(pRsp->topics);
   char    vgKey[TSDB_TOPIC_FNAME_LEN + 22];
+  tscDebug("consumer %ld update ep epoch %d to epoch %d, topic num: %d", tmq->consumerId, tmq->epoch, epoch, topicNumGet);
   SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic));
   if (newTopics == NULL) {
     return false;
@@ -928,17 +938,19 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
     taosHashClear(pHash);
     topic.topicName = strdup(pTopicEp->topic);
 
+    tscDebug("consumer %ld update topic: %s", tmq->consumerId, topic.topicName);
     int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics);
     for (int32_t j = 0; j < topicNumCur; j++) {
       // find old topic
       SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, j);
       if (pTopicCur->vgs && strcmp(pTopicCur->topicName, pTopicEp->topic) == 0) {
         int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs);
+        tscDebug("consumer %ld new vg num: %d", tmq->consumerId, vgNumCur);
         if (vgNumCur == 0) break;
         for (int32_t k = 0; k < vgNumCur; k++) {
           SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, k);
           sprintf(vgKey, "%s:%d", topic.topicName, pVgCur->vgId);
-          /*printf("epoch %d vg %d build %s\n", epoch, pVgCur->vgId, vgKey);*/
+          tscDebug("consumer %ld epoch %d vg %d build %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey);
           taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t));
         }
         break;
@@ -952,18 +964,19 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
       sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId);
       int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey));
       int64_t  offset = pVgEp->offset;
-      /*printf("epoch %d vg %d offset og to %ld\n", epoch, pVgEp->vgId, offset);*/
+      tscDebug("consumer %ld epoch %d vg %d offset og to %ld", tmq->consumerId, epoch, pVgEp->vgId, offset);
       if (pOffset != NULL) {
         offset = *pOffset;
-        /*printf("epoch %d vg %d found %s\n", epoch, pVgEp->vgId, vgKey);*/
+        tscDebug("consumer %ld epoch %d vg %d found %s", tmq->consumerId, epoch, pVgEp->vgId, vgKey);
       }
-      /*printf("epoch %d vg %d offset set to %ld\n", epoch, pVgEp->vgId, offset);*/
+      tscDebug("consumer %ld epoch %d vg %d offset set to %ld\n", tmq->consumerId, epoch, pVgEp->vgId, offset);
       SMqClientVg clientVg = {
           .pollCnt = 0,
           .currentOffset = offset,
           .vgId = pVgEp->vgId,
           .epSet = pVgEp->epSet,
           .vgStatus = TMQ_VG_STATUS__IDLE,
+          .skipCnt = 0,
       };
       taosArrayPush(topic.vgs, &clientVg);
       set = true;
@@ -981,7 +994,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
   SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param;
   tmq_t*           tmq = pParam->tmq;
   if (code != 0) {
-    tscError("get topic endpoint error, not ready, wait:%d\n", pParam->sync);
+    tscError("consumer %ld get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->sync);
     goto END;
   }
 
@@ -990,6 +1003,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
   // Epoch will only increase when received newer epoch ep msg
   SMqRspHead* head = pMsg->pData;
   int32_t     epoch = atomic_load_32(&tmq->epoch);
+  tscDebug("consumer %ld recv ep, msg epoch %d, current epoch %d", tmq->consumerId, head->epoch, epoch);
   if (head->epoch <= epoch) {
     goto END;
   }
@@ -1014,10 +1028,11 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
     tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pRsp);
 
     taosWriteQitem(tmq->mqueue, pRsp);
-    tsem_post(&tmq->rspSem);
+    /*tsem_post(&tmq->rspSem);*/
   }
 
 END:
+  atomic_store_8(&tmq->epStatus, 0);
   if (pParam->sync) {
     tsem_post(&pParam->rspSem);
   }
@@ -1025,10 +1040,18 @@ END:
 }
 
 int32_t tmqAskEp(tmq_t* tmq, bool sync) {
+  int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1);
+  if (epStatus == 1) {
+    int32_t epSkipCnt = atomic_add_fetch_32(&tmq->epSkipCnt, 1);
+    tscDebug("consumer %ld skip ask ep cnt %d", tmq->consumerId, epSkipCnt);
+    if (epSkipCnt < 5000) return 0;
+  }
+  atomic_store_32(&tmq->epSkipCnt, 0);
   int32_t           tlen = sizeof(SMqCMGetSubEpReq);
   SMqCMGetSubEpReq* req = taosMemoryMalloc(tlen);
   if (req == NULL) {
     tscError("failed to malloc get subscribe ep buf");
+    atomic_store_8(&tmq->epStatus, 0);
     return -1;
   }
   req->consumerId = htobe64(tmq->consumerId);
@@ -1039,6 +1062,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
   if (pParam == NULL) {
     tscError("failed to malloc subscribe param");
     taosMemoryFree(req);
+    atomic_store_8(&tmq->epStatus, 0);
     return -1;
   }
   pParam->tmq = tmq;
@@ -1050,6 +1074,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
     tsem_destroy(&pParam->rspSem);
     taosMemoryFree(pParam);
     taosMemoryFree(req);
+    atomic_store_8(&tmq->epStatus, 0);
     return -1;
   }
 
@@ -1067,6 +1092,8 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
 
   SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
 
+  tscDebug("consumer %ld ask ep", tmq->consumerId);
+
   int64_t transporterId = 0;
   asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
 
@@ -1205,24 +1232,34 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
       SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
       int32_t      vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
       if (vgStatus != TMQ_VG_STATUS__IDLE) {
-        /*printf("skip vg %d\n", pVg->vgId);*/
+        int64_t skipCnt = atomic_add_fetch_64(&pVg->skipCnt, 1);
+        tscDebug("consumer %ld epoch %d skip vg %d skip cnt %ld", tmq->consumerId, tmq->epoch, pVg->vgId, skipCnt);
         continue;
+#if 0
+        if (skipCnt < 30000) {
+          continue;
+        } else {
+        tscDebug("consumer %ld skip vg %d skip too much reset", tmq->consumerId, pVg->vgId);
+        }
+#endif
       }
+      atomic_store_64(&pVg->skipCnt, 0);
       SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
       if (pReq == NULL) {
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        tsem_post(&tmq->rspSem);
+        /*tsem_post(&tmq->rspSem);*/
         return -1;
       }
       SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam));
       if (pParam == NULL) {
         taosMemoryFree(pReq);
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        tsem_post(&tmq->rspSem);
+        /*tsem_post(&tmq->rspSem);*/
         return -1;
       }
       pParam->tmq = tmq;
       pParam->pVg = pVg;
+      pParam->vgId = pVg->vgId;
       pParam->epoch = tmq->epoch;
       pParam->sync = 0;
 
@@ -1231,7 +1268,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
         taosMemoryFree(pReq);
         taosMemoryFree(pParam);
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
-        tsem_post(&tmq->rspSem);
+        /*tsem_post(&tmq->rspSem);*/
         return -1;
       }
 
@@ -1249,7 +1286,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
       int64_t transporterId = 0;
       /*printf("send poll\n");*/
       atomic_add_fetch_32(&tmq->waitingRequest, 1);
-      /*tscDebug("tmq send poll: vg %d, req offset %ld", pVg->vgId, pVg->currentOffset);*/
+      tscDebug("consumer %ld send poll: vg %d, epoch %d, req offset %ld", tmq->consumerId, pVg->vgId, tmq->epoch, pVg->currentOffset);
       /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/
       asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
       pVg->pollCnt++;
@@ -1260,13 +1297,13 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
 }
 
 // return
-int32_t tmqHandleRes(tmq_t* tmq, SMqRspHead* rspHead, bool* pReset) {
+int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspHead* rspHead, bool* pReset) {
   if (rspHead->mqMsgType == TMQ_MSG_TYPE__EP_RSP) {
     /*printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);*/
     if (rspHead->epoch > atomic_load_32(&tmq->epoch)) {
       SMqCMGetSubEpRsp* rspMsg = (SMqCMGetSubEpRsp*)rspHead;
       tmqUpdateEp(tmq, rspHead->epoch, rspMsg);
-      tmqClearUnhandleMsg(tmq);
+      /*tmqClearUnhandleMsg(tmq);*/
       *pReset = true;
     } else {
       *pReset = false;
@@ -1297,6 +1334,11 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese
         /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/
         pVg->currentOffset = rspMsg->msg.rspOffset;
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
+        if (rspMsg->msg.numOfTopics == 0) {
+          taosFreeQitem(rspMsg);
+          rspHead = NULL;
+          continue;
+        }
         return rspMsg;
       } else {
         /*printf("epoch mismatch\n");*/
@@ -1305,10 +1347,10 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese
     } else {
       /*printf("handle ep rsp %d\n", rspMsg->head.mqMsgType);*/
       bool reset = false;
-      tmqHandleRes(tmq, rspHead, &reset);
+      tmqHandleNoPollRsp(tmq, rspHead, &reset);
       taosFreeQitem(rspHead);
       if (pollIfReset && reset) {
-        printf("reset and repoll\n");
+        tscDebug("consumer %ld reset and repoll", tmq->consumerId);
         tmqPollImpl(tmq, blockingTime);
       }
     }
@@ -1358,7 +1400,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
     tmqAskEp(tmq, false);
     tmqPollImpl(tmq, blocking_time);
 
-    tsem_wait(&tmq->rspSem);
+    /*tsem_wait(&tmq->rspSem);*/
 
     rspMsg = tmqHandleAllRsp(tmq, blocking_time, false);
     if (rspMsg) {
@@ -1547,24 +1589,3 @@ TAOS_ROW tmq_get_row(tmq_message_t* message) {
 }
 
 char* tmq_get_topic_name(tmq_message_t* message) { return "not implemented yet"; }
-
-#if 0
-tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) {
-  tmq_t* pTmq = taosMemoryMalloc(sizeof(tmq_t));
-  if (pTmq == NULL) {
-    return NULL;
-  }
-  strcpy(pTmq->groupId, conf->groupId);
-  strcpy(pTmq->clientId, conf->clientId);
-  pTmq->pTscObj = (STscObj*)conn;
-  pTmq->pTscObj->connType = HEARTBEAT_TYPE_MQ;
-  return pTmq;
-}
-
-
-static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) {
-  assert(pMsgBody != NULL);
-  taosMemoryFreeClear(pMsgBody->msgInfo.pData);
-  taosMemoryFreeClear(pMsgBody);
-}
-#endif

source/common/src/tdatablock.c

@@ -1139,7 +1139,7 @@ void blockDataCleanup(SSDataBlock* pDataBlock) {
   }
 }
 
-int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) {
+int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) {
   if (0 == numOfRows) {
     return TSDB_CODE_SUCCESS;
   }
@@ -1181,7 +1181,7 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
 
   for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) {
     SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
-    code = blockDataEnsureColumnCapacity(p, numOfRows);
+    code = colInfoDataEnsureCapacity(p, numOfRows);
     if (code) {
       return code;
     }
@@ -1233,6 +1233,63 @@ void colDataDestroy(SColumnInfoData* pColData) {
   taosMemoryFree(pColData->pData);
 }
 
+
+static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) {
+  int32_t len = BitmapLen(total);
+
+  int32_t newLen = BitmapLen(total - n);
+  if (n%8 == 0) {
+    memmove(nullBitmap, nullBitmap + n/8, newLen);
+  } else {
+    int32_t tail = n % 8;
+    int32_t i = 0;
+
+    uint8_t* p = (uint8_t*) nullBitmap;
+    while(i < len) {
+      uint8_t v = p[i];
+
+      p[i] = 0;
+      p[i] = (v << tail);
+
+      if (i < len - 1) {
+        uint8_t next = p[i + 1];
+        p[i] |= (next >> (8 - tail));
+      }
+
+      i += 1;
+    }
+  }
+}
+
+static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) {
+  if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
+    memmove(pColInfoData->varmeta.offset, &pColInfoData->varmeta.offset[n], (total - n));
+    memset(&pColInfoData->varmeta.offset[total - n - 1], 0, n);
+  } else {
+    int32_t bytes = pColInfoData->info.bytes;
+    memmove(pColInfoData->pData, ((char*)pColInfoData->pData + n * bytes), (total - n) * bytes);
+    doShiftBitmap(pColInfoData->nullbitmap, n, total);
+  }
+}
+
+int32_t blockDataTrimFirstNRows(SSDataBlock *pBlock, size_t n) {
+  if (n == 0) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (pBlock->info.rows <= n) {
+    blockDataCleanup(pBlock);
+  } else {
+    for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
+      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
+      colDataTrimFirstNRows(pColInfoData, n, pBlock->info.rows);
+    }
+
+    pBlock->info.rows -= n;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
   int64_t tbUid = pBlock->info.uid;
   int16_t numOfCols = pBlock->info.numOfCols;
@@ -1372,6 +1429,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
 
   return buf;
 }
+
 void blockDebugShowData(const SArray* dataBlocks) {
   char    pBuf[128];
   int32_t sz = taosArrayGetSize(dataBlocks);

source/common/src/tdataformat.c

@@ -33,7 +33,8 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
   spaceNeeded += (int)nBitmapBytes;
   // TODO: Currently, the compression of bitmap parts is affiliated to the column data parts, thus allocate 1 more
   // TYPE_BYTES as to comprise complete TYPE_BYTES. Otherwise, invalid read/write would be triggered.
-  spaceNeeded += TYPE_BYTES[pCol->type];
+  // spaceNeeded += TYPE_BYTES[pCol->type]; // the bitmap part is append as a single part since 2022.04.03, thus remove
+  // the additional space
 #endif
 
   if (pCol->spaceSize < spaceNeeded) {
@@ -47,6 +48,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
     }
   }
 #ifdef TD_SUPPORT_BITMAP
+
   if (IS_VAR_DATA_TYPE(pCol->type)) {
     pCol->pBitmap = POINTER_SHIFT(pCol->pData, pCol->bytes * maxPoints);
     pCol->dataOff = POINTER_SHIFT(pCol->pBitmap, nBitmapBytes);
@@ -85,6 +87,7 @@ int tdEncodeSchema(void **buf, STSchema *pSchema) {
   for (int i = 0; i < schemaNCols(pSchema); i++) {
     STColumn *pCol = schemaColAt(pSchema, i);
     tlen += taosEncodeFixedI8(buf, colType(pCol));
+    tlen += taosEncodeFixedI8(buf, colSma(pCol));
     tlen += taosEncodeFixedI16(buf, colColId(pCol));
     tlen += taosEncodeFixedI16(buf, colBytes(pCol));
   }
@@ -107,12 +110,14 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) {
 
   for (int i = 0; i < numOfCols; i++) {
     col_type_t  type = 0;
+    int8_t      sma = TSDB_BSMA_TYPE_NONE;
     col_id_t    colId = 0;
     col_bytes_t bytes = 0;
     buf = taosDecodeFixedI8(buf, &type);
+    buf = taosDecodeFixedI8(buf, &sma);
     buf = taosDecodeFixedI16(buf, &colId);
     buf = taosDecodeFixedI32(buf, &bytes);
-    if (tdAddColToSchema(&schemaBuilder, type, colId, bytes) < 0) {
+    if (tdAddColToSchema(&schemaBuilder, type, sma, colId, bytes) < 0) {
       tdDestroyTSchemaBuilder(&schemaBuilder);
       return NULL;
     }
@@ -148,7 +153,7 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) {
   pBuilder->version = version;
 }
 
-int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col_bytes_t bytes) {
+int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col_id_t colId, col_bytes_t bytes) {
   if (!isValidDataType(type)) return -1;
 
   if (pBuilder->nCols >= pBuilder->tCols) {
@@ -161,6 +166,7 @@ int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col
   STColumn *pCol = &(pBuilder->columns[pBuilder->nCols]);
   colSetType(pCol, type);
   colSetColId(pCol, colId);
+  colSetSma(pCol, sma);
   if (pBuilder->nCols == 0) {
     colSetOffset(pCol, 0);
   } else {
@@ -168,9 +174,6 @@ int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col
     colSetOffset(pCol, pTCol->offset + TYPE_BYTES[pTCol->type]);
   }
 
-  // TODO: set sma value by user input
-  pCol->sma = 1;
-
   if (IS_VAR_DATA_TYPE(type)) {
     colSetBytes(pCol, bytes);
     pBuilder->tlen += (TYPE_BYTES[type] + bytes);
@@ -305,7 +308,7 @@ static FORCE_INLINE const void *tdGetColDataOfRowUnsafe(SDataCol *pCol, int row)
 
 bool isNEleNull(SDataCol *pCol, int nEle) {
   if (isAllRowsNull(pCol)) return true;
-  for (int i = 0; i < nEle; i++) {
+  for (int i = 0; i < nEle; ++i) {
     if (!isNull(tdGetColDataOfRowUnsafe(pCol, i), pCol->type)) return false;
   }
   return true;
@@ -326,7 +329,7 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) {
 static void dataColSetNEleNull(SDataCol *pCol, int nEle) {
   if (IS_VAR_DATA_TYPE(pCol->type)) {
     pCol->len = 0;
-    for (int i = 0; i < nEle; i++) {
+    for (int i = 0; i < nEle; ++i) {
       dataColSetNullAt(pCol, i);
     }
   } else {
@@ -342,7 +345,7 @@ void *dataColSetOffset(SDataCol *pCol, int nEle) {
   // char *tptr = (char *)(pCol->pData);
 
   VarDataOffsetT offset = 0;
-  for (int i = 0; i < nEle; i++) {
+  for (int i = 0; i < nEle; ++i) {
     pCol->dataOff[i] = offset;
     offset += varDataTLen(tptr);
     tptr = POINTER_SHIFT(tptr, varDataTLen(tptr));
@@ -370,6 +373,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) {
       tdFreeDataCols(pCols);
       return NULL;
     }
+#if 0  // no need as calloc used
     int i;
     for (i = 0; i < maxCols; i++) {
       pCols->cols[i].spaceSize = 0;
@@ -377,6 +381,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) {
       pCols->cols[i].pData = NULL;
       pCols->cols[i].dataOff = NULL;
     }
+#endif
   }
 
   return pCols;
@@ -390,17 +395,21 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
     void *ptr = (SDataCol *)taosMemoryRealloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols);
     if (ptr == NULL) return -1;
     pCols->cols = ptr;
-    for (i = oldMaxCols; i < pCols->maxCols; i++) {
+    for (i = oldMaxCols; i < pCols->maxCols; ++i) {
       pCols->cols[i].pData = NULL;
       pCols->cols[i].dataOff = NULL;
+      pCols->cols[i].pBitmap = NULL;
       pCols->cols[i].spaceSize = 0;
     }
   }
+#if 0
+  tdResetDataCols(pCols); // redundant loop to reset len/blen to 0, already reset in following dataColInit(...)
+#endif
 
-  tdResetDataCols(pCols);
+  pCols->numOfRows = 0;
   pCols->numOfCols = schemaNCols(pSchema);
 
-  for (i = 0; i < schemaNCols(pSchema); i++) {
+  for (i = 0; i < schemaNCols(pSchema); ++i) {
     dataColInit(pCols->cols + i, schemaColAt(pSchema, i), pCols->maxPoints);
   }
 
@@ -412,7 +421,7 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) {
   if (pCols) {
     if (pCols->cols) {
       int maxCols = pCols->maxCols;
-      for (i = 0; i < maxCols; i++) {
+      for (i = 0; i < maxCols; ++i) {
         SDataCol *pCol = &pCols->cols[i];
         taosMemoryFreeClear(pCol->pData);
       }
@@ -463,7 +472,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
 void tdResetDataCols(SDataCols *pCols) {
   if (pCols != NULL) {
     pCols->numOfRows = 0;
-    for (int i = 0; i < pCols->maxCols; i++) {
+    for (int i = 0; i < pCols->maxCols; ++i) {
       dataColReset(pCols->cols + i);
     }
   }

source/common/src/tmsg.c

@@ -296,33 +296,30 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
   switch (pReq->type) {
     case TD_SUPER_TABLE:
       tlen += taosEncodeFixedI64(buf, pReq->stbCfg.suid);
-      tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nCols);
-      for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) {
+      tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nCols);
+      tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nBSmaCols);
+      for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) {
         tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].type);
+        tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].sma);
         tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pSchema[i].colId);
         tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].bytes);
         tlen += taosEncodeString(buf, pReq->stbCfg.pSchema[i].name);
       }
-      tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nTagCols);
-      for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) {
+      tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nTagCols);
+      for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) {
         tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].type);
         tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pTagSchema[i].colId);
         tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes);
         tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name);
       }
-      tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nBSmaCols);
-      for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) {
-        tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pBSmaCols[i]);
-      }
       if (pReq->rollup && pReq->stbCfg.pRSmaParam) {
         SRSmaParam *param = pReq->stbCfg.pRSmaParam;
-        tlen += taosEncodeFixedU32(buf, (uint32_t)param->xFilesFactor);
-        tlen += taosEncodeFixedI8(buf, param->delayUnit);
+        tlen += taosEncodeBinary(buf, (const void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
+        tlen += taosEncodeFixedI32(buf, param->delay);
         tlen += taosEncodeFixedI8(buf, param->nFuncIds);
         for (int8_t i = 0; i < param->nFuncIds; ++i) {
           tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]);
         }
-        tlen += taosEncodeFixedI64(buf, param->delay);
       }
       break;
     case TD_CHILD_TABLE:
@@ -330,26 +327,23 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
       tlen += tdEncodeKVRow(buf, pReq->ctbCfg.pTag);
       break;
     case TD_NORMAL_TABLE:
-      tlen += taosEncodeFixedU32(buf, pReq->ntbCfg.nCols);
-      for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) {
+      tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nCols);
+      tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nBSmaCols);
+      for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) {
         tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].type);
+        tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].sma);
         tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pSchema[i].colId);
         tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes);
         tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name);
       }
-      tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nBSmaCols);
-      for (col_id_t i = 0; i < pReq->ntbCfg.nBSmaCols; ++i) {
-        tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pBSmaCols[i]);
-      }
       if (pReq->rollup && pReq->ntbCfg.pRSmaParam) {
         SRSmaParam *param = pReq->ntbCfg.pRSmaParam;
-        tlen += taosEncodeFixedU32(buf, (uint32_t)param->xFilesFactor);
-        tlen += taosEncodeFixedI8(buf, param->delayUnit);
+        tlen += taosEncodeBinary(buf, (const void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
+        tlen += taosEncodeFixedI32(buf, param->delay);
         tlen += taosEncodeFixedI8(buf, param->nFuncIds);
         for (int8_t i = 0; i < param->nFuncIds; ++i) {
           tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]);
         }
-        tlen += taosEncodeFixedI64(buf, param->delay);
       }
       break;
     default:
@@ -370,45 +364,38 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
   switch (pReq->type) {
     case TD_SUPER_TABLE:
       buf = taosDecodeFixedI64(buf, &(pReq->stbCfg.suid));
-      buf = taosDecodeFixedU32(buf, &(pReq->stbCfg.nCols));
-      pReq->stbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchema));
-      for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) {
+      buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nCols));
+      buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols));
+      pReq->stbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchemaEx));
+      for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) {
         buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type));
+        buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].sma));
         buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.pSchema[i].colId));
         buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].bytes));
         buf = taosDecodeStringTo(buf, pReq->stbCfg.pSchema[i].name);
       }
-      buf = taosDecodeFixedU32(buf, &pReq->stbCfg.nTagCols);
+      buf = taosDecodeFixedI16(buf, &pReq->stbCfg.nTagCols);
       pReq->stbCfg.pTagSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nTagCols * sizeof(SSchema));
-      for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) {
+      for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) {
         buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type));
         buf = taosDecodeFixedI16(buf, &pReq->stbCfg.pTagSchema[i].colId);
         buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes);
         buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name);
       }
-      buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols));
-      if (pReq->stbCfg.nBSmaCols > 0) {
-        pReq->stbCfg.pBSmaCols = (col_id_t *)taosMemoryMalloc(pReq->stbCfg.nBSmaCols * sizeof(col_id_t));
-        for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) {
-          buf = taosDecodeFixedI16(buf, pReq->stbCfg.pBSmaCols + i);
-        }
-      } else {
-        pReq->stbCfg.pBSmaCols = NULL;
-      }
       if (pReq->rollup) {
         pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam));
         SRSmaParam *param = pReq->stbCfg.pRSmaParam;
-        buf = taosDecodeFixedU32(buf, (uint32_t *)&param->xFilesFactor);
-        buf = taosDecodeFixedI8(buf, &param->delayUnit);
+        buf = taosDecodeBinaryTo(buf, (void*)&param->xFilesFactor, sizeof(param->xFilesFactor));
+        buf = taosDecodeFixedI32(buf, &param->delay);
         buf = taosDecodeFixedI8(buf, &param->nFuncIds);
         if (param->nFuncIds > 0) {
+          param->pFuncIds = (func_id_t *)taosMemoryMalloc(param->nFuncIds * sizeof(func_id_t));
           for (int8_t i = 0; i < param->nFuncIds; ++i) {
             buf = taosDecodeFixedI32(buf, param->pFuncIds + i);
           }
         } else {
           param->pFuncIds = NULL;
         }
-        buf = taosDecodeFixedI64(buf, &param->delay);
       } else {
         pReq->stbCfg.pRSmaParam = NULL;
       }
@@ -418,37 +405,30 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
       buf = tdDecodeKVRow(buf, &pReq->ctbCfg.pTag);
       break;
     case TD_NORMAL_TABLE:
-      buf = taosDecodeFixedU32(buf, &pReq->ntbCfg.nCols);
-      pReq->ntbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchema));
-      for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) {
+      buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.nCols);
+      buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols));
+      pReq->ntbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchemaEx));
+      for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) {
         buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type);
+        buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].sma);
         buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.pSchema[i].colId);
         buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes);
         buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name);
       }
-      buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols));
-      if (pReq->ntbCfg.nBSmaCols > 0) {
-        pReq->ntbCfg.pBSmaCols = (col_id_t *)taosMemoryMalloc(pReq->ntbCfg.nBSmaCols * sizeof(col_id_t));
-        for (col_id_t i = 0; i < pReq->ntbCfg.nBSmaCols; ++i) {
-          buf = taosDecodeFixedI16(buf, pReq->ntbCfg.pBSmaCols + i);
-        }
-      } else {
-        pReq->ntbCfg.pBSmaCols = NULL;
-      }
       if (pReq->rollup) {
         pReq->ntbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam));
         SRSmaParam *param = pReq->ntbCfg.pRSmaParam;
-        buf = taosDecodeFixedU32(buf, (uint32_t *)&param->xFilesFactor);
-        buf = taosDecodeFixedI8(buf, &param->delayUnit);
+        buf = taosDecodeBinaryTo(buf, (void*)&param->xFilesFactor, sizeof(param->xFilesFactor));
+        buf = taosDecodeFixedI32(buf, &param->delay);
         buf = taosDecodeFixedI8(buf, &param->nFuncIds);
         if (param->nFuncIds > 0) {
+          param->pFuncIds = (func_id_t *)taosMemoryMalloc(param->nFuncIds * sizeof(func_id_t));
           for (int8_t i = 0; i < param->nFuncIds; ++i) {
             buf = taosDecodeFixedI32(buf, param->pFuncIds + i);
           }
         } else {
           param->pFuncIds = NULL;
         }
-        buf = taosDecodeFixedI64(buf, &param->delay);
       } else {
         pReq->ntbCfg.pRSmaParam = NULL;
       }
@@ -1629,6 +1609,7 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
   if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->quorum) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->replications) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -1650,6 +1631,7 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
   if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->quorum) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->replications) < 0) return -1;
   tEndDecode(&decoder);
 
   tCoderClear(&decoder);

source/common/src/tmsgcb.c

@@ -32,10 +32,6 @@ int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq) {
   return (*pMsgCb->sendReqFp)(pMsgCb->pWrapper, epSet, pReq);
 }
 
-int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, SRpcMsg* pReq) {
-  return (*pMsgCb->sendMnodeReqFp)(pMsgCb->pWrapper, pReq);
-}
-
 void tmsgSendRsp(const SRpcMsg* pRsp) { return (*tsDefaultMsgCb.sendRspFp)(tsDefaultMsgCb.pWrapper, pRsp); }
 
 void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg) {

source/common/src/trow.c

@@ -503,7 +503,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
           memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, dataOffSize);
         }
         if (!TD_COL_ROWS_NORM(pRet->cols + i)) {
-          int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->maxPoints);
+          int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->numOfRows);
           memcpy(pRet->cols[i].pBitmap, pDataCols->cols[i].pBitmap, nBitmapBytes);
         }
       }

source/common/src/ttypes.c

@@ -431,7 +431,7 @@ FORCE_INLINE void *getDataMax(int32_t type) {
   }
 }
 
-bool isValidDataType(int32_t type) { return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_UBIGINT; }
+bool isValidDataType(int32_t type) { return type >= TSDB_DATA_TYPE_NULL && type < TSDB_DATA_TYPE_MAX; }
 
 void setVardataNull(void *val, int32_t type) {
   if (type == TSDB_DATA_TYPE_BINARY) {

source/dnode/mgmt/CMakeLists.txt

-aux_source_directory(dm/src DNODE_SRC)
-aux_source_directory(qm/src DNODE_SRC)
-aux_source_directory(bm/src DNODE_SRC)
-aux_source_directory(sm/src DNODE_SRC)
-aux_source_directory(vm/src DNODE_SRC)
-aux_source_directory(mm/src DNODE_SRC)
-aux_source_directory(main/src DNODE_SRC)
+aux_source_directory(dm DNODE_SRC)
+aux_source_directory(qm DNODE_SRC)
+aux_source_directory(bm DNODE_SRC)
+aux_source_directory(sm DNODE_SRC)
+aux_source_directory(vm DNODE_SRC)
+aux_source_directory(mm DNODE_SRC)
+aux_source_directory(main DNODE_SRC)
 add_library(dnode STATIC ${DNODE_SRC})
 target_link_libraries(
     dnode cjson mnode vnode qnode snode bnode wal sync taos tfs monitor
@@ -12,20 +12,14 @@ target_link_libraries(
 target_include_directories(
     dnode
     PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mgmt"
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/dm/inc"
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/qm/inc"
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/bm/inc"
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/sm/inc"
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/vm/inc"
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/mm/inc"
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/main/inc"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 
-aux_source_directory(main/exe EXEC_SRC)
+aux_source_directory(exe EXEC_SRC)
 add_executable(taosd ${EXEC_SRC})
 target_include_directories(
     taosd
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/main/inc"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 target_link_libraries(taosd dnode)
 

source/dnode/mgmt/bm/src/bmMsg.c → source/dnode/mgmt/bm/bmHandle.c

@@ -54,4 +54,4 @@ int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
   }
 }
 
-void bmInitMsgHandles(SMgmtWrapper *pWrapper) {}
+void bmInitMsgHandle(SMgmtWrapper *pWrapper) {}

source/dnode/mgmt/bm/src/bmInt.c → source/dnode/mgmt/bm/bmInt.c

@@ -19,12 +19,7 @@
 static int32_t bmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); }
 
 static void bmInitOption(SBnodeMgmt *pMgmt, SBnodeOpt *pOption) {
-  SMsgCb msgCb = {0};
-  msgCb.pWrapper = pMgmt->pWrapper;
-  msgCb.sendReqFp = dndSendReqToDnode;
-  msgCb.sendMnodeReqFp = dndSendReqToMnode;
-  msgCb.sendRspFp = dndSendRsp;
-  msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
+  SMsgCb msgCb = dndCreateMsgcb(pMgmt->pWrapper);
   pOption->msgCb = msgCb;
 }
 
@@ -114,7 +109,7 @@ int32_t bmOpen(SMgmtWrapper *pWrapper) {
   return code;
 }
 
-void bmGetMgmtFp(SMgmtWrapper *pWrapper) {
+void bmSetMgmtFp(SMgmtWrapper *pWrapper) {
   SMgmtFp mgmtFp = {0};
   mgmtFp.openFp = bmOpen;
   mgmtFp.closeFp = bmClose;
@@ -122,7 +117,7 @@ void bmGetMgmtFp(SMgmtWrapper *pWrapper) {
   mgmtFp.dropMsgFp = bmProcessDropReq;
   mgmtFp.requiredFp = bmRequire;
 
-  bmInitMsgHandles(pWrapper);
+  bmInitMsgHandle(pWrapper);
   pWrapper->name = "bnode";
   pWrapper->fp = mgmtFp;
 }

source/dnode/mgmt/bm/src/bmWorker.c → source/dnode/mgmt/bm/bmWorker.c

@@ -18,7 +18,7 @@
 
 static void bmSendErrorRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
   SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code};
-  dndSendRsp(pWrapper, &rpcRsp);
+  tmsgSendRsp(&rpcRsp);
 
   dTrace("msg:%p, is freed", pMsg);
   rpcFreeCont(pMsg->rpcMsg.pCont);

source/dnode/mgmt/bm/inc/bm.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_DND_BNODE_H_
-#define _TD_DND_BNODE_H_
-
-#include "dnd.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-void bmGetMgmtFp(SMgmtWrapper *pWrapper);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_DND_BNODE_H_*/
\ No newline at end of file

source/dnode/mgmt/dm/src/dmFile.c → source/dnode/mgmt/dm/dmFile.c

@@ -200,7 +200,7 @@ int32_t dmWriteFile(SDnodeMgmt *pMgmt) {
   taosMemoryFree(content);
 
   char realfile[PATH_MAX];
-  snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP);
+  snprintf(realfile, sizeof(realfile), "%s%sdnode.json", pMgmt->path, TD_DIRSEP);
 
   if (taosRenameFile(file, realfile) != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);

source/dnode/mgmt/dm/src/dmMsg.c → source/dnode/mgmt/dm/dmHandle.c

@@ -15,7 +15,6 @@
 
 #define _DEFAULT_SOURCE
 #include "dmInt.h"
-#include "vm.h"
 
 void dmSendStatusReq(SDnodeMgmt *pMgmt) {
   SDnode    *pDnode = pMgmt->pDnode;
@@ -57,7 +56,9 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
   pMgmt->statusSent = 1;
 
   dTrace("send req:%s to mnode, app:%p", TMSG_INFO(rpcMsg.msgType), rpcMsg.ahandle);
-  dndSendReqToMnode(pMgmt->pWrapper, &rpcMsg);
+  SEpSet epSet = {0};
+  dmGetMnodeEpSet(pMgmt, &epSet);
+  tmsgSendReq(&pMgmt->msgCb, &epSet, &rpcMsg);
 }
 
 static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) {
@@ -116,21 +117,99 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) {
   return TSDB_CODE_OPS_NOT_SUPPORT;
 }
 
-void dmInitMsgHandles(SMgmtWrapper *pWrapper) {
+
+static int32_t dmProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) {
+  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype);
+  if (pWrapper != NULL) {
+    dndReleaseWrapper(pWrapper);
+    terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED;
+    dError("failed to create node since %s", terrstr());
+    return -1;
+  }
+
+  pWrapper = &pDnode->wrappers[ntype];
+
+  if (taosMkDir(pWrapper->path) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to create dir:%s since %s", pWrapper->path, terrstr());
+    return -1;
+  }
+
+  int32_t code = (*pWrapper->fp.createMsgFp)(pWrapper, pMsg);
+  if (code != 0) {
+    dError("node:%s, failed to open since %s", pWrapper->name, terrstr());
+  } else {
+    dDebug("node:%s, has been opened", pWrapper->name);
+    pWrapper->deployed = true;
+  }
+
+  return code;
+}
+
+static int32_t dmProcessDropNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) {
+  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype);
+  if (pWrapper == NULL) {
+    terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
+    dError("failed to drop node since %s", terrstr());
+    return -1;
+  }
+
+  taosWLockLatch(&pWrapper->latch);
+  pWrapper->deployed = false;
+
+  int32_t code = (*pWrapper->fp.dropMsgFp)(pWrapper, pMsg);
+  if (code != 0) {
+    pWrapper->deployed = true;
+    dError("node:%s, failed to drop since %s", pWrapper->name, terrstr());
+  } else {
+    pWrapper->deployed = false;
+    dDebug("node:%s, has been dropped", pWrapper->name);
+  }
+
+  taosWUnLockLatch(&pWrapper->latch);
+  dndReleaseWrapper(pWrapper);
+  return code;
+}
+
+int32_t dmProcessCDnodeReq(SDnode *pDnode, SNodeMsg *pMsg) {
+  switch (pMsg->rpcMsg.msgType) {
+    case TDMT_DND_CREATE_MNODE:
+      return dmProcessCreateNodeMsg(pDnode, MNODE, pMsg);
+    case TDMT_DND_DROP_MNODE:
+      return dmProcessDropNodeMsg(pDnode, MNODE, pMsg);
+    case TDMT_DND_CREATE_QNODE:
+      return dmProcessCreateNodeMsg(pDnode, QNODE, pMsg);
+    case TDMT_DND_DROP_QNODE:
+      return dmProcessDropNodeMsg(pDnode, QNODE, pMsg);
+    case TDMT_DND_CREATE_SNODE:
+      return dmProcessCreateNodeMsg(pDnode, SNODE, pMsg);
+    case TDMT_DND_DROP_SNODE:
+      return dmProcessDropNodeMsg(pDnode, SNODE, pMsg);
+    case TDMT_DND_CREATE_BNODE:
+      return dmProcessCreateNodeMsg(pDnode, BNODE, pMsg);
+    case TDMT_DND_DROP_BNODE:
+      return dmProcessDropNodeMsg(pDnode, BNODE, pMsg);
+    default:
+      terrno = TSDB_CODE_MSG_NOT_PROCESSED;
+      return -1;
+  }
+}
+
+void dmInitMsgHandle(SMgmtWrapper *pWrapper) {
   // Requests handled by DNODE
-  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, dmProcessMgmtMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, dmProcessMgmtMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, dmProcessMgmtMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, dmProcessMgmtMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, dmProcessMgmtMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, dmProcessMgmtMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, dmProcessMgmtMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, dmProcessMgmtMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, dmProcessMgmtMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, dmProcessMgmtMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, dmProcessMgmtMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, dmProcessMgmtMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, dmProcessMgmtMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, dmProcessMgmtMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, dmProcessMgmtMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, dmProcessMgmtMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, dmProcessMgmtMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, dmProcessMgmtMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, dmProcessMgmtMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, dmProcessMgmtMsg, DEFAULT_HANDLE);
 
   // Requests handled by MNODE
-  dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, dmProcessMgmtMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, dmProcessMgmtMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, dmProcessMgmtMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, dmProcessStatusMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, dmProcessMgmtMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, dmProcessMgmtMsg, DEFAULT_HANDLE);
 }

source/dnode/mgmt/dm/src/dmInt.c → source/dnode/mgmt/dm/dmInt.c

@@ -78,7 +78,7 @@ static int32_t dmStart(SMgmtWrapper *pWrapper) {
   return dmStartThread(pWrapper->pMgmt);
 }
 
-int32_t dmInit(SMgmtWrapper *pWrapper) {
+static int32_t dmInit(SMgmtWrapper *pWrapper) {
   SDnode     *pDnode = pWrapper->pDnode;
   SDnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SDnodeMgmt));
   dInfo("dnode-mgmt start to init");
@@ -112,22 +112,19 @@ int32_t dmInit(SMgmtWrapper *pWrapper) {
     return -1;
   }
 
-  if (dndInitServer(pDnode) != 0) {
-    dError("failed to init trans server since %s", terrstr());
-    return -1;
-  }
-
-  if (dndInitClient(pDnode) != 0) {
-    dError("failed to init trans client since %s", terrstr());
+  if (dndInitTrans(pDnode) != 0) {
+    dError("failed to init transport since %s", terrstr());
     return -1;
   }
 
   pWrapper->pMgmt = pMgmt;
+  pMgmt->msgCb = dndCreateMsgcb(pWrapper);
+
   dInfo("dnode-mgmt is initialized");
   return 0;
 }
 
-void dmCleanup(SMgmtWrapper *pWrapper) {
+static void dmCleanup(SMgmtWrapper *pWrapper) {
   SDnodeMgmt *pMgmt = pWrapper->pMgmt;
   if (pMgmt == NULL) return;
 
@@ -151,25 +148,24 @@ void dmCleanup(SMgmtWrapper *pWrapper) {
 
   taosMemoryFree(pMgmt);
   pWrapper->pMgmt = NULL;
-  dndCleanupServer(pDnode);
-  dndCleanupClient(pDnode);
+  dndCleanupTrans(pDnode);
 
   dInfo("dnode-mgmt is cleaned up");
 }
 
-int32_t dmRequire(SMgmtWrapper *pWrapper, bool *required) {
+static int32_t dmRequire(SMgmtWrapper *pWrapper, bool *required) {
   *required = true;
   return 0;
 }
 
-void dmGetMgmtFp(SMgmtWrapper *pWrapper) {
+void dmSetMgmtFp(SMgmtWrapper *pWrapper) {
   SMgmtFp mgmtFp = {0};
   mgmtFp.openFp = dmInit;
   mgmtFp.closeFp = dmCleanup;
   mgmtFp.startFp = dmStart;
   mgmtFp.requiredFp = dmRequire;
 
-  dmInitMsgHandles(pWrapper);
+  dmInitMsgHandle(pWrapper);
   pWrapper->name = "dnode";
   pWrapper->fp = mgmtFp;
 }

source/dnode/mgmt/main/src/dndMonitor.c → source/dnode/mgmt/dm/dmMonitor.c

@@ -16,7 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "dndInt.h"
 
-static int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) {
+static int32_t dmGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) {
   tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name));
   pInfo->logdir.size = tsLogSpace.size;
   tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name));
@@ -30,14 +30,14 @@ static int32_t dndGetMonitorDiskInfo(SDnode *pDnode, SMonDiskInfo *pInfo) {
   return 0;
 }
 
-static void dndGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
+static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
   pInfo->protocol = 1;
   pInfo->dnode_id = pDnode->dnodeId;
   pInfo->cluster_id = pDnode->clusterId;
   tstrncpy(pInfo->dnode_ep, tsLocalEp, TSDB_EP_LEN);
 }
 
-static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
+static void dmGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
   pInfo->uptime = (taosGetTimestampMs() - pDnode->rebootTime) / (86400000.0f);
   taosGetCpuUsage(&pInfo->cpu_engine, &pInfo->cpu_system);
   taosGetCpuCores(&pInfo->cpu_cores);
@@ -63,7 +63,7 @@ static void dndGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) {
   }
 }
 
-void dndSendMonitorReport(SDnode *pDnode) {
+void dmSendMonitorReport(SDnode *pDnode) {
   if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return;
   dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort);
 
@@ -71,7 +71,7 @@ void dndSendMonitorReport(SDnode *pDnode) {
   if (pMonitor == NULL) return;
 
   SMonBasicInfo basicInfo = {0};
-  dndGetMonitorBasicInfo(pDnode, &basicInfo);
+  dmGetMonitorBasicInfo(pDnode, &basicInfo);
   monSetBasicInfo(pMonitor, &basicInfo);
 
   SMonClusterInfo clusterInfo = {0};
@@ -89,11 +89,11 @@ void dndSendMonitorReport(SDnode *pDnode) {
   }
 
   SMonDnodeInfo dnodeInfo = {0};
-  dndGetMonitorDnodeInfo(pDnode, &dnodeInfo);
+  dmGetMonitorDnodeInfo(pDnode, &dnodeInfo);
   monSetDnodeInfo(pMonitor, &dnodeInfo);
 
   SMonDiskInfo diskInfo = {0};
-  if (dndGetMonitorDiskInfo(pDnode, &diskInfo) == 0) {
+  if (dmGetMonitorDiskInfo(pDnode, &diskInfo) == 0) {
     monSetDiskInfo(pMonitor, &diskInfo);
   }
 

source/dnode/mgmt/dm/src/dmWorker.c → source/dnode/mgmt/dm/dmWorker.c

@@ -14,16 +14,11 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "bm.h"
 #include "dmInt.h"
-#include "mm.h"
-#include "qm.h"
-#include "sm.h"
-#include "vm.h"
 
 static void *dmThreadRoutine(void *param) {
   SDnodeMgmt *pMgmt = param;
-  SDnode *    pDnode = pMgmt->pDnode;
+  SDnode     *pDnode = pMgmt->pDnode;
   int64_t     lastStatusTime = taosGetTimestampMs();
   int64_t     lastMonitorTime = lastStatusTime;
 
@@ -37,8 +32,7 @@ static void *dmThreadRoutine(void *param) {
     }
 
     int64_t curTime = taosGetTimestampMs();
-
-    float statusInterval = (curTime - lastStatusTime) / 1000.0f;
+    float   statusInterval = (curTime - lastStatusTime) / 1000.0f;
     if (statusInterval >= tsStatusInterval && !pMgmt->statusSent) {
       dmSendStatusReq(pMgmt);
       lastStatusTime = curTime;
@@ -46,31 +40,32 @@ static void *dmThreadRoutine(void *param) {
 
     float monitorInterval = (curTime - lastMonitorTime) / 1000.0f;
     if (monitorInterval >= tsMonitorInterval) {
-      dndSendMonitorReport(pDnode);
+      dmSendMonitorReport(pDnode);
       lastMonitorTime = curTime;
     }
   }
 }
 
+int32_t dmStartThread(SDnodeMgmt *pMgmt) {
+  pMgmt->threadId = taosCreateThread(dmThreadRoutine, pMgmt);
+  if (pMgmt->threadId == NULL) {
+    dError("failed to init dnode thread");
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  return 0;
+}
+
 static void dmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
   SDnodeMgmt *pMgmt = pInfo->ahandle;
 
-  SDnode * pDnode = pMgmt->pDnode;
+  SDnode  *pDnode = pMgmt->pDnode;
   SRpcMsg *pRpc = &pMsg->rpcMsg;
   int32_t  code = -1;
   dTrace("msg:%p, will be processed in dnode queue", pMsg);
 
   switch (pRpc->msgType) {
-    case TDMT_DND_CREATE_MNODE:
-    case TDMT_DND_CREATE_QNODE:
-    case TDMT_DND_CREATE_SNODE:
-    case TDMT_DND_CREATE_BNODE:
-    case TDMT_DND_DROP_MNODE:
-    case TDMT_DND_DROP_QNODE:
-    case TDMT_DND_DROP_SNODE:
-    case TDMT_DND_DROP_BNODE:
-      code = dndProcessNodeMsg(pMgmt->pDnode, pMsg);
-      break;
     case TDMT_DND_CONFIG_DNODE:
       code = dmProcessConfigReq(pMgmt, pMsg);
       break;
@@ -84,8 +79,8 @@ static void dmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
       code = dmProcessGrantRsp(pMgmt, pMsg);
       break;
     default:
-      terrno = TSDB_CODE_MSG_NOT_PROCESSED;
-      dError("msg:%p, type:%s not processed in dnode queue", pRpc->handle, TMSG_INFO(pRpc->msgType));
+      code = dmProcessCDnodeReq(pMgmt->pDnode, pMsg);
+      break;
   }
 
   if (pRpc->msgType & 1u) {
@@ -100,16 +95,14 @@ static void dmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
 }
 
 int32_t dmStartWorker(SDnodeMgmt *pMgmt) {
-  SSingleWorkerCfg mgmtCfg = {
-      .min = 1, .max = 1, .name = "dnode-mgmt", .fp = (FItem)dmProcessQueue, .param = pMgmt};
-  if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mgmtCfg) != 0) {
+  SSingleWorkerCfg mcfg = {.min = 1, .max = 1, .name = "dnode-mgmt", .fp = (FItem)dmProcessQueue, .param = pMgmt};
+  if (tSingleWorkerInit(&pMgmt->mgmtWorker, &mcfg) != 0) {
     dError("failed to start dnode mgmt worker since %s", terrstr());
     return -1;
   }
 
-  SSingleWorkerCfg statusCfg = {
-      .min = 1, .max = 1, .name = "dnode-status", .fp = (FItem)dmProcessQueue, .param = pMgmt};
-  if (tSingleWorkerInit(&pMgmt->statusWorker, &statusCfg) != 0) {
+  SSingleWorkerCfg scfg = {.min = 1, .max = 1, .name = "dnode-status", .fp = (FItem)dmProcessQueue, .param = pMgmt};
+  if (tSingleWorkerInit(&pMgmt->statusWorker, &scfg) != 0) {
     dError("failed to start dnode status worker since %s", terrstr());
     return -1;
   }
@@ -118,17 +111,6 @@ int32_t dmStartWorker(SDnodeMgmt *pMgmt) {
   return 0;
 }
 
-int32_t dmStartThread(SDnodeMgmt *pMgmt) {
-  pMgmt->threadId = taosCreateThread(dmThreadRoutine, pMgmt);
-  if (pMgmt->threadId == NULL) {
-    dError("failed to init dnode thread");
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  return 0;
-}
-
 void dmStopWorker(SDnodeMgmt *pMgmt) {
   tSingleWorkerCleanup(&pMgmt->mgmtWorker);
   tSingleWorkerCleanup(&pMgmt->statusWorker);
@@ -143,9 +125,15 @@ void dmStopWorker(SDnodeMgmt *pMgmt) {
 int32_t dmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
   SDnodeMgmt    *pMgmt = pWrapper->pMgmt;
   SSingleWorker *pWorker = &pMgmt->mgmtWorker;
-  if (pMsg->rpcMsg.msgType == TDMT_MND_STATUS_RSP) {
-    pWorker = &pMgmt->statusWorker;
-  }
+
+  dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
+  taosWriteQitem(pWorker->queue, pMsg);
+  return 0;
+}
+
+int32_t dmProcessStatusMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
+  SDnodeMgmt    *pMgmt = pWrapper->pMgmt;
+  SSingleWorker *pWorker = &pMgmt->statusWorker;
 
   dTrace("msg:%p, put into worker %s", pMsg, pWorker->name);
   taosWriteQitem(pWorker->queue, pMsg);

source/dnode/mgmt/dm/inc/dm.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_DND_DNODE_H_
-#define _TD_DND_DNODE_H_
-
-#include "dnd.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct SDnodeMgmt SDnodeMgmt;
-
-void dmGetMgmtFp(SMgmtWrapper *pWrapper);
-void dmInitMsgHandles(SMgmtWrapper *pWrapper);
-
-void    dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet);
-void    dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet);
-void    dmSendRedirectRsp(SDnodeMgmt *pMgmt, const SRpcMsg *pMsg);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_DND_DNODE_H_*/
\ No newline at end of file

source/dnode/mgmt/main/exe/dndMain.c → source/dnode/mgmt/exe/dndMain.c

@@ -14,7 +14,7 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "dnd.h"
+#include "dndInt.h"
 #include "tconfig.h"
 
 static struct {
@@ -30,7 +30,6 @@ static struct {
 } global = {0};
 
 static void dndStopDnode(int signum, void *info, void *ctx) {
-  dInfo("system signal:%d received", signum);
   SDnode *pDnode = atomic_val_compare_exchange_ptr(&global.pDnode, 0, global.pDnode);
   if (pDnode != NULL) {
     dndHandleEvent(pDnode, DND_EVENT_STOP);
@@ -41,8 +40,10 @@ static void dndSetSignalHandle() {
   taosSetSignal(SIGTERM, dndStopDnode);
   taosSetSignal(SIGHUP, dndStopDnode);
   taosSetSignal(SIGINT, dndStopDnode);
+  taosSetSignal(SIGTSTP, dndStopDnode);
   taosSetSignal(SIGABRT, dndStopDnode);
   taosSetSignal(SIGBREAK, dndStopDnode);
+  taosSetSignal(SIGQUIT, dndStopDnode);
 
   if (!tsMultiProcess) {
   } else if (global.ntype == DNODE || global.ntype == NODE_MAX) {
@@ -72,7 +73,7 @@ static int32_t dndParseArgs(int32_t argc, char const *argv[]) {
     } else if (strcmp(argv[i], "-n") == 0) {
       global.ntype = atoi(argv[++i]);
       if (global.ntype <= DNODE || global.ntype > NODE_MAX) {
-        printf("'-n' range is [1-5], default is 0\n");
+        printf("'-n' range is [1 - %d], default is 0\n", NODE_MAX - 1);
         return -1;
       }
     } else if (strcmp(argv[i], "-k") == 0) {

source/dnode/mgmt/bm/inc/bmInt.h → source/dnode/mgmt/inc/bmInt.h

@@ -16,7 +16,7 @@
 #ifndef _TD_DND_BNODE_INT_H_
 #define _TD_DND_BNODE_INT_H_
 
-#include "bm.h"
+#include "dndInt.h"
 #include "bnode.h"
 
 #ifdef __cplusplus
@@ -35,8 +35,8 @@ typedef struct SBnodeMgmt {
 int32_t bmOpen(SMgmtWrapper *pWrapper);
 int32_t bmDrop(SMgmtWrapper *pWrapper);
 
-// bmMsg.c
-void    bmInitMsgHandles(SMgmtWrapper *pWrapper);
+// bmHandle.c
+void    bmInitMsgHandle(SMgmtWrapper *pWrapper);
 int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t bmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 

source/dnode/mgmt/dm/inc/dmInt.h → source/dnode/mgmt/inc/dmInt.h

@@ -16,7 +16,7 @@
 #ifndef _TD_DND_DNODE_INT_H_
 #define _TD_DND_DNODE_INT_H_
 
-#include "dm.h"
+#include "dndInt.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -33,6 +33,7 @@ typedef struct SDnodeMgmt {
   SRWLatch      latch;
   SSingleWorker mgmtWorker;
   SSingleWorker statusWorker;
+  SMsgCb        msgCb;
   const char   *path;
   SDnode       *pDnode;
   SMgmtWrapper *pWrapper;
@@ -43,18 +44,24 @@ int32_t dmReadFile(SDnodeMgmt *pMgmt);
 int32_t dmWriteFile(SDnodeMgmt *pMgmt);
 void    dmUpdateDnodeEps(SDnodeMgmt *pMgmt, SArray *pDnodeEps);
 
-// dmMsg.c
+// dmHandle.c
+void    dmInitMsgHandle(SMgmtWrapper *pWrapper);
 void    dmSendStatusReq(SDnodeMgmt *pMgmt);
 int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg);
 int32_t dmProcessStatusRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg);
 int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg);
 int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SNodeMsg *pMsg);
+int32_t dmProcessCDnodeReq(SDnode *pDnode, SNodeMsg *pMsg);
+
+// dmMonitor.c
+void dmSendMonitorReport(SDnode *pDnode);
 
 // dmWorker.c
+int32_t dmStartThread(SDnodeMgmt *pMgmt);
 int32_t dmStartWorker(SDnodeMgmt *pMgmt);
 void    dmStopWorker(SDnodeMgmt *pMgmt);
-int32_t dmStartThread(SDnodeMgmt *pMgmt);
 int32_t dmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+int32_t dmProcessStatusMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 
 #ifdef __cplusplus
 }

source/dnode/mgmt/main/inc/dnd.h → source/dnode/mgmt/inc/dndInt.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_DND_H_
-#define _TD_DND_H_
+#ifndef _TD_DND_INT_H_
+#define _TD_DND_INT_H_
 
 #include "os.h"
 
@@ -27,13 +27,13 @@
 #include "tlockfree.h"
 #include "tlog.h"
 #include "tmsg.h"
+#include "tmsgcb.h"
 #include "tprocess.h"
 #include "tqueue.h"
 #include "trpc.h"
 #include "tthread.h"
 #include "ttime.h"
 #include "tworker.h"
-#include "tmsgcb.h"
 
 #include "dnode.h"
 #include "monitor.h"
@@ -73,7 +73,7 @@ typedef int32_t (*DropNodeFp)(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 typedef int32_t (*RequireNodeFp)(SMgmtWrapper *pWrapper, bool *required);
 
 typedef struct SMsgHandle {
-  SMgmtWrapper *pQndWrapper; 
+  SMgmtWrapper *pQndWrapper;
   SMgmtWrapper *pMndWrapper;
   SMgmtWrapper *pWrapper;
 } SMsgHandle;
@@ -92,6 +92,7 @@ typedef struct SMgmtWrapper {
   char       *path;
   int32_t     refCount;
   SRWLatch    latch;
+  ENodeType   ntype;
   bool        deployed;
   bool        required;
   EProcType   procType;
@@ -134,35 +135,72 @@ typedef struct SDnode {
   SMgmtWrapper wrappers[NODE_MAX];
 } SDnode;
 
+// dndEnv.c
+const char *dndStatStr(EDndStatus stat);
 const char *dndNodeLogStr(ENodeType ntype);
 const char *dndNodeProcStr(ENodeType ntype);
 const char *dndEventStr(EDndEvent ev);
-EDndStatus  dndGetStatus(SDnode *pDnode);
-void        dndSetStatus(SDnode *pDnode, EDndStatus stat);
-void        dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId);
-void        dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc);
-void        dndSendMonitorReport(SDnode *pDnode);
-
-int32_t dndInitServer(SDnode *pDnode);
-void    dndCleanupServer(SDnode *pDnode);
-int32_t dndInitClient(SDnode *pDnode);
-void    dndCleanupClient(SDnode *pDnode);
-int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg);
-int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
-int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pMsg);
-void    dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp);
-void    dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
-SMsgCb  dndCreateMsgcb(SMgmtWrapper *pWrapper);
-
-int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed);
-int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed);
 
+// dndExec.c
+int32_t dndOpenNode(SMgmtWrapper *pWrapper);
+void    dndCloseNode(SMgmtWrapper *pWrapper);
+
+// dndFile.c
+int32_t   dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed);
+int32_t   dndWriteFile(SMgmtWrapper *pWrapper, bool deployed);
+TdFilePtr dndCheckRunning(const char *dataDir);
+int32_t   dndReadShmFile(SDnode *pDnode);
+int32_t   dndWriteShmFile(SDnode *pDnode);
+
+// dndInt.c
+EDndStatus    dndGetStatus(SDnode *pDnode);
+void          dndSetStatus(SDnode *pDnode, EDndStatus stat);
+void          dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId);
 SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType);
 int32_t       dndMarkWrapper(SMgmtWrapper *pWrapper);
 void          dndReleaseWrapper(SMgmtWrapper *pWrapper);
+void          dndHandleEvent(SDnode *pDnode, EDndEvent event);
+void          dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc);
+void          dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg);
+
+// dndTransport.c
+int32_t  dndInitTrans(SDnode *pDnode);
+void     dndCleanupTrans(SDnode *pDnode);
+SMsgCb   dndCreateMsgcb(SMgmtWrapper *pWrapper);
+SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper);
+int32_t  dndInitMsgHandle(SDnode *pDnode);
+
+// mgmt
+void dmSetMgmtFp(SMgmtWrapper *pWrapper);
+void bmSetMgmtFp(SMgmtWrapper *pWrapper);
+void qmSetMgmtFp(SMgmtWrapper *pMgmt);
+void smSetMgmtFp(SMgmtWrapper *pWrapper);
+void vmSetMgmtFp(SMgmtWrapper *pWrapper);
+void mmSetMgmtFp(SMgmtWrapper *pMgmt);
+
+void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet);
+void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet);
+void dmSendRedirectRsp(SDnodeMgmt *pMgmt, const SRpcMsg *pMsg);
+
+typedef struct {
+  int32_t openVnodes;
+  int32_t totalVnodes;
+  int32_t masterNum;
+  int64_t numOfSelectReqs;
+  int64_t numOfInsertReqs;
+  int64_t numOfInsertSuccessReqs;
+  int64_t numOfBatchInsertReqs;
+  int64_t numOfBatchInsertSuccessReqs;
+} SVnodesStat;
+
+void    vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads);
+int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo);
+void    vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo);
+int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
+                           SMonGrantInfo *pGrantInfo);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_DND_H_*/
+#endif /*_TD_DND_INT_H_*/
\ No newline at end of file

source/dnode/mgmt/mm/inc/mmInt.h → source/dnode/mgmt/inc/mmInt.h

@@ -16,7 +16,7 @@
 #ifndef _TD_DND_MNODE_INT_H_
 #define _TD_DND_MNODE_INT_H_
 
-#include "mm.h"
+#include "dndInt.h"
 #include "mnode.h"
 
 #ifdef __cplusplus
@@ -46,8 +46,8 @@ int32_t mmOpenFromMsg(SMgmtWrapper *pWrapper, SDCreateMnodeReq *pReq);
 int32_t mmDrop(SMgmtWrapper *pWrapper);
 int32_t mmAlter(SMnodeMgmt *pMgmt, SDAlterMnodeReq *pReq);
 
-// mmMsg.c
-void    mmInitMsgHandles(SMgmtWrapper *pWrapper);
+// mmHandle.c
+void    mmInitMsgHandle(SMgmtWrapper *pWrapper);
 int32_t mmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t mmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg);

source/dnode/mgmt/qm/inc/qmInt.h → source/dnode/mgmt/inc/qmInt.h

@@ -16,7 +16,7 @@
 #ifndef _TD_DND_QNODE_INT_H_
 #define _TD_DND_QNODE_INT_H_
 
-#include "qm.h"
+#include "dndInt.h"
 #include "qnode.h"
 
 #ifdef __cplusplus
@@ -36,8 +36,8 @@ typedef struct SQnodeMgmt {
 int32_t qmOpen(SMgmtWrapper *pWrapper);
 int32_t qmDrop(SMgmtWrapper *pWrapper);
 
-// qmMsg.c
-void    qmInitMsgHandles(SMgmtWrapper *pWrapper);
+// qmHandle.c
+void    qmInitMsgHandle(SMgmtWrapper *pWrapper);
 int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 

source/dnode/mgmt/sm/inc/smInt.h → source/dnode/mgmt/inc/smInt.h

@@ -16,7 +16,7 @@
 #ifndef _TD_DND_SNODE_INT_H_
 #define _TD_DND_SNODE_INT_H_
 
-#include "sm.h"
+#include "dndInt.h"
 #include "snode.h"
 
 #ifdef __cplusplus
@@ -38,8 +38,8 @@ typedef struct SSnodeMgmt {
 int32_t smOpen(SMgmtWrapper *pWrapper);
 int32_t smDrop(SMgmtWrapper *pWrapper);
 
-// smMsg.c
-void    smInitMsgHandles(SMgmtWrapper *pWrapper);
+// smHandle.c
+void    smInitMsgHandle(SMgmtWrapper *pWrapper);
 int32_t smProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 

source/dnode/mgmt/vm/inc/vmInt.h → source/dnode/mgmt/inc/vmInt.h

@@ -17,7 +17,7 @@
 #define _TD_DND_VNODES_INT_H_
 
 #include "sync.h"
-#include "vm.h"
+#include "dndInt.h"
 #include "vnode.h"
 
 #ifdef __cplusplus
@@ -84,8 +84,8 @@ void       vmReleaseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode);
 int32_t    vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl);
 void       vmCloseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode);
 
-// vmMsg.c
-void    vmInitMsgHandles(SMgmtWrapper *pWrapper);
+// vmHandle.c
+void    vmInitMsgHandle(SMgmtWrapper *pWrapper);
 int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq);
 int32_t vmProcessAlterVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq);
 int32_t vmProcessDropVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pReq);

source/dnode/mgmt/main/src/dndStr.c → source/dnode/mgmt/main/dndEnv.c

@@ -15,6 +15,48 @@
 
 #define _DEFAULT_SOURCE
 #include "dndInt.h"
+#include "wal.h"
+
+static int8_t once = DND_ENV_INIT;
+
+int32_t dndInit() {
+  dDebug("start to init dnode env");
+  if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) {
+    terrno = TSDB_CODE_REPEAT_INIT;
+    dError("failed to init dnode env since %s", terrstr());
+    return -1;
+  }
+
+  taosIgnSIGPIPE();
+  taosBlockSIGPIPE();
+  taosResolveCRC();
+
+  SMonCfg monCfg = {0};
+  monCfg.maxLogs = tsMonitorMaxLogs;
+  monCfg.port = tsMonitorPort;
+  monCfg.server = tsMonitorFqdn;
+  monCfg.comp = tsMonitorComp;
+  if (monInit(&monCfg) != 0) {
+    dError("failed to init monitor since %s", terrstr());
+    return -1;
+  }
+
+  dInfo("dnode env is initialized");
+  return 0;
+}
+
+void dndCleanup() {
+  dDebug("start to cleanup dnode env");
+  if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) {
+    dError("dnode env is already cleaned up");
+    return;
+  }
+
+  monCleanup();
+  walCleanUp();
+  taosStopCacheRefreshWorker();
+  dInfo("dnode env is cleaned up");
+}
 
 const char *dndStatStr(EDndStatus status) {
   switch (status) {
@@ -25,7 +67,7 @@ const char *dndStatStr(EDndStatus status) {
     case DND_STAT_STOPPED:
       return "stopped";
     default:
-      return "unknown";
+      return "UNKNOWN";
   }
 }
 

source/dnode/mgmt/main/src/dndExec.c → source/dnode/mgmt/main/dndExec.c

@@ -20,9 +20,9 @@ static bool dndRequireNode(SMgmtWrapper *pWrapper) {
   bool    required = false;
   int32_t code = (*pWrapper->fp.requiredFp)(pWrapper, &required);
   if (!required) {
-    dDebug("node:%s, no need to start", pWrapper->name);
+    dDebug("node:%s, does not require startup", pWrapper->name);
   } else {
-    dDebug("node:%s, need to start", pWrapper->name);
+    dDebug("node:%s, needs to be started", pWrapper->name);
   }
   return required;
 }
@@ -45,7 +45,7 @@ int32_t dndOpenNode(SMgmtWrapper *pWrapper) {
 }
 
 void dndCloseNode(SMgmtWrapper *pWrapper) {
-  dDebug("node:%s, start to close", pWrapper->name);
+  dDebug("node:%s, mgmt start to close", pWrapper->name);
   pWrapper->required = false;
   taosWLockLatch(&pWrapper->latch);
   if (pWrapper->deployed) {
@@ -62,53 +62,9 @@ void dndCloseNode(SMgmtWrapper *pWrapper) {
     taosProcCleanup(pWrapper->pProc);
     pWrapper->pProc = NULL;
   }
-  dDebug("node:%s, has been closed", pWrapper->name);
+  dDebug("node:%s, mgmt has been closed", pWrapper->name);
 }
 
-static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen,
-                                 ProcFuncType ftype) {
-  SRpcMsg *pRpc = &pMsg->rpcMsg;
-  pRpc->pCont = pCont;
-  dTrace("msg:%p, get from child queue, handle:%p app:%p", pMsg, pRpc->handle, pRpc->ahandle);
-
-  NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)];
-  int32_t   code = (*msgFp)(pWrapper, pMsg);
-
-  if (code != 0) {
-    dError("msg:%p, failed to process since code:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
-    if (pRpc->msgType & 1U) {
-      SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno};
-      dndSendRsp(pWrapper, &rsp);
-    }
-
-    dTrace("msg:%p, is freed", pMsg);
-    taosFreeQitem(pMsg);
-    rpcFreeCont(pCont);
-  }
-}
-
-static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen,
-                                  ProcFuncType ftype) {
-  pMsg->pCont = pCont;
-  dTrace("msg:%p, get from parent queue, handle:%p app:%p", pMsg, pMsg->handle, pMsg->ahandle);
-
-  switch (ftype) {
-    case PROC_REG:
-      rpcRegisterBrokenLinkArg(pMsg);
-      break;
-    case PROC_RELEASE:
-      rpcReleaseHandle(pMsg->handle, (int8_t)pMsg->code);
-      rpcFreeCont(pCont);
-      break;
-    case PROC_REQ:
-      // todo send to dnode
-      dndSendReqToMnode(pWrapper, pMsg);
-    default:
-      dndSendRpcRsp(pWrapper, pMsg);
-      break;
-  }
-  taosMemoryFree(pMsg);
-}
 
 static int32_t dndNewProc(SMgmtWrapper *pWrapper, ENodeType n) {
   char  tstr[8] = {0};
@@ -132,21 +88,9 @@ static int32_t dndNewProc(SMgmtWrapper *pWrapper, ENodeType n) {
   return 0;
 }
 
-static SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper) {
-  SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)dndConsumeChildQueue,
-                  .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem,
-                  .childFreeHeadFp = (ProcFreeFp)taosFreeQitem,
-                  .childMallocBodyFp = (ProcMallocFp)rpcMallocCont,
-                  .childFreeBodyFp = (ProcFreeFp)rpcFreeCont,
-                  .parentConsumeFp = (ProcConsumeFp)dndConsumeParentQueue,
-                  .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc,
-                  .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree,
-                  .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont,
-                  .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont,
-                  .shm = pWrapper->shm,
-                  .pParent = pWrapper,
-                  .name = pWrapper->name};
-  return cfg;
+static void dndProcessProcHandle(void *handle) {
+  SRpcMsg rpcMsg = {.handle = handle, .code = TSDB_CODE_DND_OFFLINE};
+  rpcSendResponse(&rpcMsg);
 }
 
 static int32_t dndRunInSingleProcess(SDnode *pDnode) {
@@ -180,6 +124,7 @@ static int32_t dndRunInSingleProcess(SDnode *pDnode) {
   while (1) {
     if (pDnode->event == DND_EVENT_STOP) {
       dInfo("dnode is about to stop");
+      dndSetStatus(pDnode, DND_STAT_STOPPED);
       break;
     }
     taosMsleep(100);
@@ -202,7 +147,7 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) {
     if (!pWrapper->required) continue;
 
     int32_t shmsize = 1024 * 1024 * 2;  // size will be a configuration item
-    if (taosCreateShm(&pWrapper->shm, shmsize) != 0) {
+    if (taosCreateShm(&pWrapper->shm, n, shmsize) != 0) {
       terrno = TAOS_SYSTEM_ERROR(terrno);
       dError("node:%s, failed to create shm size:%d since %s", pWrapper->name, shmsize, terrstr());
       return -1;
@@ -255,17 +200,33 @@ static int32_t dndRunInParentProcess(SDnode *pDnode) {
   while (1) {
     if (pDnode->event == DND_EVENT_STOP) {
       dInfo("dnode is about to stop");
+      dndSetStatus(pDnode, DND_STAT_STOPPED);
+
+      for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) {
+        SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+        if (!pWrapper->required) continue;
+        if (pDnode->ntype == NODE_MAX) continue;
+
+        if (pWrapper->procId > 0 && taosProcExist(pWrapper->procId)) {
+          dInfo("node:%s, send kill signal to the child process:%d", pWrapper->name, pWrapper->procId);
+          taosKillProc(pWrapper->procId);
+          dInfo("node:%s, wait for child process:%d to stop", pWrapper->name, pWrapper->procId);
+          taosWaitProc(pWrapper->procId);
+          dInfo("node:%s, child process:%d is stopped", pWrapper->name, pWrapper->procId);
+        }
+      }
       break;
-    }
-
-    for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) {
-      SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
-      if (!pWrapper->required) continue;
-      if (pDnode->ntype == NODE_MAX) continue;
-
-      if (pWrapper->procId <= 0 || !taosProcExists(pWrapper->procId)) {
-        dInfo("node:%s, process:%d is killed and needs to be restarted", pWrapper->name, pWrapper->procId);
-        dndNewProc(pWrapper, n);
+    } else {
+      for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) {
+        SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+        if (!pWrapper->required) continue;
+        if (pDnode->ntype == NODE_MAX) continue;
+
+        if (pWrapper->procId <= 0 || !taosProcExist(pWrapper->procId)) {
+          dInfo("node:%s, process:%d is killed and needs to be restarted", pWrapper->name, pWrapper->procId);
+          taosProcCloseHandles(pWrapper->pProc, dndProcessProcHandle);
+          dndNewProc(pWrapper, n);
+        }
       }
     }
 
@@ -279,6 +240,12 @@ static int32_t dndRunInChildProcess(SDnode *pDnode) {
   SMgmtWrapper *pWrapper = &pDnode->wrappers[pDnode->ntype];
   dInfo("%s run in child process", pWrapper->name);
 
+  pWrapper->required = dndRequireNode(pWrapper);
+  if (!pWrapper->required) {
+    dError("%s does not require startup", pWrapper->name);
+    return -1;
+  }
+
   SMsgCb msgCb = dndCreateMsgcb(pWrapper);
   tmsgSetDefaultMsgCb(&msgCb);
   pWrapper->procType = PROC_CHILD;
@@ -303,6 +270,8 @@ static int32_t dndRunInChildProcess(SDnode *pDnode) {
     }
   }
 
+  dndSetStatus(pDnode, DND_STAT_RUNNING);
+
   if (taosProcRun(pWrapper->pProc) != 0) {
     dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr());
     return -1;
@@ -312,11 +281,14 @@ static int32_t dndRunInChildProcess(SDnode *pDnode) {
   dndReportStartup(pDnode, "TDengine", "initialized successfully");
   while (1) {
     if (pDnode->event == DND_EVENT_STOP) {
-      dInfo("dnode is about to stop");
+      dInfo("%s is about to stop", pWrapper->name);
+      dndSetStatus(pDnode, DND_STAT_STOPPED);
       break;
     }
     taosMsleep(100);
   }
+
+  return 0;
 }
 
 int32_t dndRun(SDnode *pDnode) {

source/dnode/mgmt/main/src/dndInt.c → source/dnode/mgmt/main/dndInt.c

@@ -82,12 +82,12 @@ SDnode *dndCreate(const SDnodeOpt *pOption) {
   }
 
   dndSetStatus(pDnode, DND_STAT_INIT);
-  dmGetMgmtFp(&pDnode->wrappers[DNODE]);
-  mmGetMgmtFp(&pDnode->wrappers[MNODE]);
-  vmGetMgmtFp(&pDnode->wrappers[VNODES]);
-  qmGetMgmtFp(&pDnode->wrappers[QNODE]);
-  smGetMgmtFp(&pDnode->wrappers[SNODE]);
-  bmGetMgmtFp(&pDnode->wrappers[BNODE]);
+  dmSetMgmtFp(&pDnode->wrappers[DNODE]);
+  mmSetMgmtFp(&pDnode->wrappers[MNODE]);
+  vmSetMgmtFp(&pDnode->wrappers[VNODES]);
+  qmSetMgmtFp(&pDnode->wrappers[QNODE]);
+  smSetMgmtFp(&pDnode->wrappers[SNODE]);
+  bmSetMgmtFp(&pDnode->wrappers[BNODE]);
 
   for (ENodeType n = 0; n < NODE_MAX; ++n) {
     SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
@@ -95,6 +95,7 @@ SDnode *dndCreate(const SDnodeOpt *pOption) {
     pWrapper->path = strdup(path);
     pWrapper->shm.id = -1;
     pWrapper->pDnode = pDnode;
+    pWrapper->ntype = n;
     if (pWrapper->path == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       goto _OVER;
@@ -133,14 +134,6 @@ _OVER:
 void dndClose(SDnode *pDnode) {
   if (pDnode == NULL) return;
 
-  if (dndGetStatus(pDnode) == DND_STAT_STOPPED) {
-    dError("dnode is shutting down, data:%p", pDnode);
-    return;
-  }
-
-  dInfo("start to close dnode, data:%p", pDnode);
-  dndSetStatus(pDnode, DND_STAT_STOPPED);
-
   for (ENodeType n = 0; n < NODE_MAX; ++n) {
     SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
     dndCloseNode(pWrapper);
@@ -151,7 +144,6 @@ void dndClose(SDnode *pDnode) {
 }
 
 void dndHandleEvent(SDnode *pDnode, EDndEvent event) {
-  dInfo("dnode receive %s event, data:%p", dndEventStr(event), pDnode);
   if (event == DND_EVENT_STOP) {
     pDnode->event = event;
   }
@@ -197,4 +189,41 @@ void dndReleaseWrapper(SMgmtWrapper *pWrapper) {
   int32_t refCount = atomic_sub_fetch_32(&pWrapper->refCount, 1);
   taosRUnLockLatch(&pWrapper->latch);
   dTrace("node:%s, is released, refCount:%d", pWrapper->name, refCount);
+}
+
+void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId) {
+  pWrapper->msgFps[TMSG_INDEX(msgType)] = nodeMsgFp;
+  pWrapper->msgVgIds[TMSG_INDEX(msgType)] = vgId;
+}
+
+EDndStatus dndGetStatus(SDnode *pDnode) { return pDnode->status; }
+
+void dndSetStatus(SDnode *pDnode, EDndStatus status) {
+  if (pDnode->status != status) {
+    dDebug("dnode status set from %s to %s", dndStatStr(pDnode->status), dndStatStr(status));
+    pDnode->status = status;
+  }
+}
+
+void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) {
+  SStartupReq *pStartup = &pDnode->startup;
+  tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN);
+  tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN);
+  pStartup->finished = 0;
+}
+
+static void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) {
+  memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq));
+  pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING);
+}
+
+void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) {
+  dDebug("startup req is received");
+  SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq));
+  dndGetStartup(pDnode, pStartup);
+
+  dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished);
+  SRpcMsg rpcRsp = {
+      .handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq), .ahandle = pReq->ahandle};
+  rpcSendResponse(&rpcRsp);
 }
\ No newline at end of file

source/dnode/mgmt/main/inc/dndInt.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_DND_INT_H_
-#define _TD_DND_INT_H_
-
-#include "dnd.h"
-
-#include "bm.h"
-#include "dm.h"
-#include "mm.h"
-#include "qm.h"
-#include "sm.h"
-#include "vm.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-// dndInt.c
-int32_t     dndInit();
-void        dndCleanup();
-const char *dndStatStr(EDndStatus stat);
-void        dndGetStartup(SDnode *pDnode, SStartupReq *pStartup);
-void        dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg);
-
-// dndMsg.c
-void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, SEpSet *pEpSet);
-
-// dndExec.c
-int32_t dndOpenNode(SMgmtWrapper *pWrapper);
-void    dndCloseNode(SMgmtWrapper *pWrapper);
-int32_t dndRun(SDnode *pDnode);
-
-// dndObj.c
-SDnode *dndCreate(const SDnodeOpt *pOption);
-void    dndClose(SDnode *pDnode);
-void    dndHandleEvent(SDnode *pDnode, EDndEvent event);
-
-// dndTransport.c
-int32_t dndInitMsgHandle(SDnode *pDnode);
-void    dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp);
-
-// dndFile.c
-TdFilePtr dndCheckRunning(const char *dataDir);
-int32_t   dndReadShmFile(SDnode *pDnode);
-int32_t   dndWriteShmFile(SDnode *pDnode);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_DND_INT_H_*/
\ No newline at end of file

source/dnode/mgmt/main/src/dndEnv.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dndInt.h"
-#include "wal.h"
-
-static int8_t once = DND_ENV_INIT;
-
-int32_t dndInit() {
-  dDebug("start to init dnode env");
-  if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) {
-    terrno = TSDB_CODE_REPEAT_INIT;
-    dError("failed to init dnode env since %s", terrstr());
-    return -1;
-  }
-
-  taosIgnSIGPIPE();
-  taosBlockSIGPIPE();
-  taosResolveCRC();
-
-  SMonCfg monCfg = {0};
-  monCfg.maxLogs = tsMonitorMaxLogs;
-  monCfg.port = tsMonitorPort;
-  monCfg.server = tsMonitorFqdn;
-  monCfg.comp = tsMonitorComp;
-  if (monInit(&monCfg) != 0) {
-    dError("failed to init monitor since %s", terrstr());
-    return -1;
-  }
-
-  dInfo("dnode env is initialized");
-  return 0;
-}
-
-void dndCleanup() {
-  dDebug("start to cleanup dnode env");
-  if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) {
-    dError("dnode env is already cleaned up");
-    return;
-  }
-
-  monCleanup();
-  walCleanUp();
-  taosStopCacheRefreshWorker();
-  dInfo("dnode env is cleaned up");
-}
-
-void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId) {
-  pWrapper->msgFps[TMSG_INDEX(msgType)] = nodeMsgFp;
-  pWrapper->msgVgIds[TMSG_INDEX(msgType)] = vgId;
-}
-
-EDndStatus dndGetStatus(SDnode *pDnode) { return pDnode->status; }
-
-void dndSetStatus(SDnode *pDnode, EDndStatus status) {
-  if (pDnode->status != status) {
-    dDebug("dnode status set from %s to %s", dndStatStr(pDnode->status), dndStatStr(status));
-    pDnode->status = status;
-  }
-}
-
-void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) {
-  SStartupReq *pStartup = &pDnode->startup;
-  tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN);
-  tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN);
-  pStartup->finished = 0;
-}
-
-void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) {
-  memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq));
-  pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING);
-}
-
-void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) {
-  dDebug("startup req is received");
-  SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq));
-  dndGetStartup(pDnode, pStartup);
-
-  dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished);
-  SRpcMsg rpcRsp = {
-      .handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq), .ahandle = pReq->ahandle};
-  rpcSendResponse(&rpcRsp);
-}

source/dnode/mgmt/main/src/dndMsg.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dndInt.h"
-
-static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) {
-  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE);
-  if (pWrapper != NULL) {
-    dmUpdateMnodeEpSet(pWrapper->pMgmt, pEpSet);
-    dndReleaseWrapper(pWrapper);
-  }
-}
-
-static inline NodeMsgFp dndGetMsgFp(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
-  NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)];
-  if (msgFp == NULL) {
-    terrno = TSDB_CODE_MSG_NOT_PROCESSED;
-  }
-
-  return msgFp;
-}
-
-static inline int32_t dndBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) {
-  SRpcConnInfo connInfo = {0};
-  if ((pRpc->msgType & 1U) && rpcGetConnInfo(pRpc->handle, &connInfo) != 0) {
-    terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
-    dError("failed to build msg since %s, app:%p RPC:%p", terrstr(), pRpc->ahandle, pRpc->handle);
-    return -1;
-  }
-
-  memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN);
-  pMsg->clientIp = connInfo.clientIp;
-  pMsg->clientPort = connInfo.clientPort;
-  memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg));
-  return 0;
-}
-
-void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) {
-  int32_t   code = -1;
-  SNodeMsg *pMsg = NULL;
-  NodeMsgFp msgFp = NULL;
-
-  if (pEpSet && pEpSet->numOfEps > 0 && pRpc->msgType == TDMT_MND_STATUS_RSP) {
-    dndUpdateMnodeEpSet(pWrapper->pDnode, pEpSet);
-  }
-
-  if (dndMarkWrapper(pWrapper) != 0) goto _OVER;
-  if ((msgFp = dndGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER;
-  if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER;
-  if (dndBuildMsg(pMsg, pRpc) != 0) goto _OVER;
-
-  if (pWrapper->procType == PROC_SINGLE) {
-    dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user);
-    code = (*msgFp)(pWrapper, pMsg);
-  } else if (pWrapper->procType == PROC_PARENT) {
-    dTrace("msg:%p, is created and put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle,
-           pRpc->ahandle, pMsg->user);
-    code = taosProcPutToChildQ(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, PROC_REQ);
-  } else {
-    dTrace("msg:%p, should not processed in child process, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle,
-           pMsg->user);
-    ASSERT(1);
-  }
-
-_OVER:
-  if (code == 0) {
-    if (pWrapper->procType == PROC_PARENT) {
-      dTrace("msg:%p, is freed in parent process", pMsg);
-      taosFreeQitem(pMsg);
-      rpcFreeCont(pRpc->pCont);
-    }
-  } else {
-    dError("msg:%p, failed to process since 0x%04x:%s", pMsg, code & 0XFFFF, terrstr());
-    if (pRpc->msgType & 1U) {
-      SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno};
-      dndSendRsp(pWrapper, &rsp);
-    }
-    dTrace("msg:%p, is freed", pMsg);
-    taosFreeQitem(pMsg);
-    rpcFreeCont(pRpc->pCont);
-  }
-
-  dndReleaseWrapper(pWrapper);
-}
-
-static int32_t dndProcessCreateNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) {
-  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype);
-  if (pWrapper != NULL) {
-    dndReleaseWrapper(pWrapper);
-    terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED;
-    dError("failed to create node since %s", terrstr());
-    return -1;
-  }
-
-  pWrapper = &pDnode->wrappers[ntype];
-
-  if (taosMkDir(pWrapper->path) != 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    dError("failed to create dir:%s since %s", pWrapper->path, terrstr());
-    return -1;
-  }
-
-  int32_t code = (*pWrapper->fp.createMsgFp)(pWrapper, pMsg);
-  if (code != 0) {
-    dError("node:%s, failed to open since %s", pWrapper->name, terrstr());
-  } else {
-    dDebug("node:%s, has been opened", pWrapper->name);
-    pWrapper->deployed = true;
-  }
-
-  return code;
-}
-
-static int32_t dndProcessDropNodeMsg(SDnode *pDnode, ENodeType ntype, SNodeMsg *pMsg) {
-  SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, ntype);
-  if (pWrapper == NULL) {
-    terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
-    dError("failed to drop node since %s", terrstr());
-    return -1;
-  }
-
-  taosWLockLatch(&pWrapper->latch);
-  pWrapper->deployed = false;
-
-  int32_t code = (*pWrapper->fp.dropMsgFp)(pWrapper, pMsg);
-  if (code != 0) {
-    pWrapper->deployed = true;
-    dError("node:%s, failed to drop since %s", pWrapper->name, terrstr());
-  } else {
-    pWrapper->deployed = false;
-    dDebug("node:%s, has been dropped", pWrapper->name);
-  }
-
-  taosWUnLockLatch(&pWrapper->latch);
-  dndReleaseWrapper(pWrapper);
-  return code;
-}
-
-int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg) {
-  switch (pMsg->rpcMsg.msgType) {
-    case TDMT_DND_CREATE_MNODE:
-      return dndProcessCreateNodeMsg(pDnode, MNODE, pMsg);
-    case TDMT_DND_DROP_MNODE:
-      return dndProcessDropNodeMsg(pDnode, MNODE, pMsg);
-    case TDMT_DND_CREATE_QNODE:
-      return dndProcessCreateNodeMsg(pDnode, QNODE, pMsg);
-    case TDMT_DND_DROP_QNODE:
-      return dndProcessDropNodeMsg(pDnode, QNODE, pMsg);
-    case TDMT_DND_CREATE_SNODE:
-      return dndProcessCreateNodeMsg(pDnode, SNODE, pMsg);
-    case TDMT_DND_DROP_SNODE:
-      return dndProcessDropNodeMsg(pDnode, SNODE, pMsg);
-    case TDMT_DND_CREATE_BNODE:
-      return dndProcessCreateNodeMsg(pDnode, BNODE, pMsg);
-    case TDMT_DND_DROP_BNODE:
-      return dndProcessDropNodeMsg(pDnode, BNODE, pMsg);
-    default:
-      terrno = TSDB_CODE_MSG_NOT_PROCESSED;
-      return -1;
-  }
-}
\ No newline at end of file

source/dnode/mgmt/mm/inc/mm.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_DND_MNODE_H_
-#define _TD_DND_MNODE_H_
-
-#include "dnd.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-void mmGetMgmtFp(SMgmtWrapper *pMgmt);
-
-int32_t mmGetUserAuth(SMgmtWrapper *pWrapper, char *user, char *spi, char *encrypt, char *secret, char *ckey);
-int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
-                         SMonGrantInfo *pGrantInfo);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_DND_MNODE_H_*/
\ No newline at end of file

source/dnode/mgmt/mm/src/mmFile.c → source/dnode/mgmt/mm/mmFile.c

@@ -111,7 +111,7 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, bool deployed) {
 
   TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
-    terrno = TAOS_SYSTEM_ERROR(errno);;
+    terrno = TAOS_SYSTEM_ERROR(errno);
     dError("failed to write %s since %s", file, terrstr());
     return -1;
   }
@@ -145,7 +145,7 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, bool deployed) {
   snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP);
 
   if (taosRenameFile(file, realfile) != 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);;
+    terrno = TAOS_SYSTEM_ERROR(errno);
     dError("failed to rename %s since %s", file, terrstr());
     return -1;
   }

source/dnode/mgmt/mm/src/mmMsg.c → source/dnode/mgmt/mm/mmHandle.c

@@ -73,93 +73,93 @@ int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) {
   }
 }
 
-void mmInitMsgHandles(SMgmtWrapper *pWrapper) {
+void mmInitMsgHandle(SMgmtWrapper *pWrapper) {
   // Requests handled by DNODE
-  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, mmProcessWriteMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
 
   // Requests handled by MNODE
-  dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, mmProcessReadMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, mmProcessReadMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SMA, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SMA, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, mmProcessReadMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, mmProcessReadMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, mmProcessReadMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, mmProcessReadMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_SYSTABLE_RETRIEVE, mmProcessReadMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, mmProcessReadMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, mmProcessReadMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, mmProcessReadMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, mmProcessReadMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, mmProcessReadMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SMA, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SMA, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, mmProcessReadMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, mmProcessReadMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, mmProcessReadMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, mmProcessReadMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_SYSTABLE_RETRIEVE, mmProcessReadMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, mmProcessReadMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, mmProcessReadMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, mmProcessReadMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
 
   // Requests handled by VNODE
-  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA_RSP, mmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA_RSP, mmProcessWriteMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
 
-  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, mmProcessQueryMsg, MND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, mmProcessQueryMsg, MND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, mmProcessQueryMsg, MND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, mmProcessQueryMsg, MND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, mmProcessQueryMsg, MND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, mmProcessQueryMsg, MNODE_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, mmProcessQueryMsg, MNODE_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, mmProcessQueryMsg, MNODE_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, mmProcessQueryMsg, MNODE_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, mmProcessQueryMsg, MNODE_HANDLE);
 
 }

source/dnode/mgmt/mm/src/mmInt.c → source/dnode/mgmt/mm/mmInt.c

@@ -39,20 +39,11 @@ static int32_t mmRequire(SMgmtWrapper *pWrapper, bool *required) {
 }
 
 static void mmInitOption(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
-  SDnode *pDnode = pMgmt->pDnode;
-  pOption->dnodeId = pDnode->dnodeId;
-  pOption->clusterId = pDnode->clusterId;
-
-  SMsgCb msgCb = {0};
-  msgCb.pWrapper = pMgmt->pWrapper;
+  SMsgCb msgCb = dndCreateMsgcb(pMgmt->pWrapper);
   msgCb.queueFps[QUERY_QUEUE] = mmPutMsgToQueryQueue;
   msgCb.queueFps[READ_QUEUE] = mmPutMsgToReadQueue;
   msgCb.queueFps[WRITE_QUEUE] = mmPutMsgToWriteQueue;
   msgCb.queueFps[SYNC_QUEUE] = mmPutMsgToWriteQueue;
-  msgCb.sendReqFp = dndSendReqToDnode;
-  msgCb.sendMnodeReqFp = dndSendReqToMnode;
-  msgCb.sendRspFp = dndSendRsp;
-  msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
   pOption->msgCb = msgCb;
 }
 
@@ -66,6 +57,7 @@ static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
   pReplica->id = 1;
   pReplica->port = pDnode->serverPort;
   tstrncpy(pReplica->fqdn, pDnode->localFqdn, TSDB_FQDN_LEN);
+  pOption->deploy = true;
 
   pMgmt->selfIndex = pOption->selfIndex;
   pMgmt->replica = pOption->replica;
@@ -77,6 +69,7 @@ static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
   pOption->selfIndex = pMgmt->selfIndex;
   pOption->replica = pMgmt->replica;
   memcpy(&pOption->replicas, pMgmt->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA);
+  pOption->deploy = false;
 }
 
 static int32_t mmBuildOptionFromReq(SMnodeMgmt *pMgmt, SMnodeOpt *pOption, SDCreateMnodeReq *pCreate) {
@@ -89,7 +82,7 @@ static int32_t mmBuildOptionFromReq(SMnodeMgmt *pMgmt, SMnodeOpt *pOption, SDCre
     pReplica->id = pCreate->replicas[i].id;
     pReplica->port = pCreate->replicas[i].port;
     memcpy(pReplica->fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN);
-    if (pReplica->id == pOption->dnodeId) {
+    if (pReplica->id == pMgmt->pDnode->dnodeId) {
       pOption->selfIndex = i;
     }
   }
@@ -98,6 +91,7 @@ static int32_t mmBuildOptionFromReq(SMnodeMgmt *pMgmt, SMnodeOpt *pOption, SDCre
     dError("failed to build mnode options since %s", terrstr());
     return -1;
   }
+  pOption->deploy = true;
 
   pMgmt->selfIndex = pOption->selfIndex;
   pMgmt->replica = pOption->replica;
@@ -225,9 +219,7 @@ int32_t mmOpenFromMsg(SMgmtWrapper *pWrapper, SDCreateMnodeReq *pReq) {
   return code;
 }
 
-static int32_t mmOpen(SMgmtWrapper *pWrapper) {
-  return mmOpenFromMsg(pWrapper, NULL);
-}
+static int32_t mmOpen(SMgmtWrapper *pWrapper) { return mmOpenFromMsg(pWrapper, NULL); }
 
 static int32_t mmStart(SMgmtWrapper *pWrapper) {
   dDebug("mnode-mgmt start to run");
@@ -235,7 +227,7 @@ static int32_t mmStart(SMgmtWrapper *pWrapper) {
   return mndStart(pMgmt->pMnode);
 }
 
-void mmGetMgmtFp(SMgmtWrapper *pWrapper) {
+void mmSetMgmtFp(SMgmtWrapper *pWrapper) {
   SMgmtFp mgmtFp = {0};
   mgmtFp.openFp = mmOpen;
   mgmtFp.closeFp = mmClose;
@@ -244,21 +236,13 @@ void mmGetMgmtFp(SMgmtWrapper *pWrapper) {
   mgmtFp.dropMsgFp = mmProcessDropReq;
   mgmtFp.requiredFp = mmRequire;
 
-  mmInitMsgHandles(pWrapper);
+  mmInitMsgHandle(pWrapper);
   pWrapper->name = "mnode";
   pWrapper->fp = mgmtFp;
 }
 
-int32_t mmGetUserAuth(SMgmtWrapper *pWrapper, char *user, char *spi, char *encrypt, char *secret, char *ckey) {
-  SMnodeMgmt *pMgmt = pWrapper->pMgmt;
-
-  int32_t code = mndRetriveAuth(pMgmt->pMnode, user, spi, encrypt, secret, ckey);
-  dTrace("user:%s, retrieve auth spi:%d encrypt:%d", user, *spi, *encrypt);
-  return code;
-}
-
 int32_t mmMonitorMnodeInfo(SMgmtWrapper *pWrapper, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
-                         SMonGrantInfo *pGrantInfo) {
+                           SMonGrantInfo *pGrantInfo) {
   SMnodeMgmt *pMgmt = pWrapper->pMgmt;
   return mndGetMonitorInfo(pMgmt->pMnode, pClusterInfo, pVgroupInfo, pGrantInfo);
 }

source/dnode/mgmt/mm/src/mmWorker.c → source/dnode/mgmt/mm/mmWorker.c

@@ -37,7 +37,7 @@ static void mmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
         dError("msg:%p, failed to process since %s", pMsg, terrstr());
       }
       SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .contLen = pMsg->rspLen, .pCont = pMsg->pRsp};
-      dndSendRsp(pMgmt->pWrapper, &rsp);
+      tmsgSendRsp(&rsp);
     }
   }
 
@@ -60,7 +60,7 @@ static void mmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
     if (pRpc->handle != NULL && code != 0) {
       dError("msg:%p, failed to process since %s", pMsg, terrstr());
       SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .ahandle = pRpc->ahandle};
-      dndSendRsp(pMgmt->pWrapper, &rsp);
+      tmsgSendRsp(&rsp);
     }
   }
 

source/dnode/mgmt/qm/inc/qm.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_DND_QNODE_H_
-#define _TD_DND_QNODE_H_
-
-#include "dnd.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-void qmGetMgmtFp(SMgmtWrapper *pMgmt);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_DND_QNODE_H_*/
\ No newline at end of file

source/dnode/mgmt/qm/src/qmMsg.c → source/dnode/mgmt/qm/qmHandle.c

@@ -54,16 +54,16 @@ int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
   }
 }
 
-void qmInitMsgHandles(SMgmtWrapper *pWrapper) {
+void qmInitMsgHandle(SMgmtWrapper *pWrapper) {
   // Requests handled by VNODE
-  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, qmProcessQueryMsg, QND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, qmProcessQueryMsg, QND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, qmProcessFetchMsg, QND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, qmProcessFetchMsg, QND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, qmProcessQueryMsg, QNODE_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, qmProcessQueryMsg, QNODE_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, qmProcessFetchMsg, QNODE_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, qmProcessFetchMsg, QNODE_HANDLE);
 
-  dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, qmProcessFetchMsg, QND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, qmProcessFetchMsg, QND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, qmProcessFetchMsg, QND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, qmProcessFetchMsg, QND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, qmProcessFetchMsg, QND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, qmProcessFetchMsg, QNODE_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, qmProcessFetchMsg, QNODE_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, qmProcessFetchMsg, QNODE_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, qmProcessFetchMsg, QNODE_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, qmProcessFetchMsg, QNODE_HANDLE);
 }

source/dnode/mgmt/qm/src/qmInt.c → source/dnode/mgmt/qm/qmInt.c

@@ -19,15 +19,10 @@
 static int32_t qmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); }
 
 static void qmInitOption(SQnodeMgmt *pMgmt, SQnodeOpt *pOption) {
-  SMsgCb msgCb = {0};
-  msgCb.pWrapper = pMgmt->pWrapper;
+  SMsgCb msgCb = dndCreateMsgcb(pMgmt->pWrapper);
   msgCb.queueFps[QUERY_QUEUE] = qmPutMsgToQueryQueue;
   msgCb.queueFps[FETCH_QUEUE] = qmPutMsgToFetchQueue;
   msgCb.qsizeFp = qmGetQueueSize;
-  msgCb.sendReqFp = dndSendReqToDnode;
-  msgCb.sendMnodeReqFp = dndSendReqToMnode;
-  msgCb.sendRspFp = dndSendRsp;
-  msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
   pOption->msgCb = msgCb;
 }
 
@@ -117,7 +112,7 @@ int32_t qmOpen(SMgmtWrapper *pWrapper) {
   return code;
 }
 
-void qmGetMgmtFp(SMgmtWrapper *pWrapper) {
+void qmSetMgmtFp(SMgmtWrapper *pWrapper) {
   SMgmtFp mgmtFp = {0};
   mgmtFp.openFp = qmOpen;
   mgmtFp.closeFp = qmClose;
@@ -125,7 +120,7 @@ void qmGetMgmtFp(SMgmtWrapper *pWrapper) {
   mgmtFp.dropMsgFp = qmProcessDropReq;
   mgmtFp.requiredFp = qmRequire;
 
-  qmInitMsgHandles(pWrapper);
+  qmInitMsgHandle(pWrapper);
   pWrapper->name = "qnode";
   pWrapper->fp = mgmtFp;
 }

source/dnode/mgmt/qm/src/qmWorker.c → source/dnode/mgmt/qm/qmWorker.c

@@ -18,7 +18,7 @@
 
 static void qmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
   SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code};
-  dndSendRsp(pWrapper, &rsp);
+  tmsgSendRsp(&rsp);
 }
 
 static void qmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {

source/dnode/mgmt/sm/inc/sm.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_DND_SNODE_H_
-#define _TD_DND_SNODE_H_
-
-#include "dnd.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-void smGetMgmtFp(SMgmtWrapper *pWrapper);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_DND_SNODE_H_*/

source/dnode/mgmt/sm/src/smMsg.c → source/dnode/mgmt/sm/smHandle.c

@@ -54,8 +54,8 @@ int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
   }
 }
 
-void smInitMsgHandles(SMgmtWrapper *pWrapper) {
+void smInitMsgHandle(SMgmtWrapper *pWrapper) {
   // Requests handled by SNODE
-  dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, smProcessMgmtMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, smProcessExecMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, smProcessMgmtMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, smProcessExecMsg, DEFAULT_HANDLE);
 }

source/dnode/mgmt/sm/src/smInt.c → source/dnode/mgmt/sm/smInt.c

@@ -19,12 +19,7 @@
 static int32_t smRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); }
 
 static void smInitOption(SSnodeMgmt *pMgmt, SSnodeOpt *pOption) {
-  SMsgCb msgCb = {0};
-  msgCb.pWrapper = pMgmt->pWrapper;
-  msgCb.sendReqFp = dndSendReqToDnode;
-  msgCb.sendMnodeReqFp = dndSendReqToMnode;
-  msgCb.sendRspFp = dndSendRsp;
-  msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
+  SMsgCb msgCb = dndCreateMsgcb(pMgmt->pWrapper);
   pOption->msgCb = msgCb;
 }
 
@@ -114,7 +109,7 @@ int32_t smOpen(SMgmtWrapper *pWrapper) {
   return code;
 }
 
-void smGetMgmtFp(SMgmtWrapper *pWrapper) {
+void smSetMgmtFp(SMgmtWrapper *pWrapper) {
   SMgmtFp mgmtFp = {0};
   mgmtFp.openFp = smOpen;
   mgmtFp.closeFp = smClose;
@@ -122,7 +117,7 @@ void smGetMgmtFp(SMgmtWrapper *pWrapper) {
   mgmtFp.dropMsgFp = smProcessDropReq;
   mgmtFp.requiredFp = smRequire;
 
-  smInitMsgHandles(pWrapper);
+  smInitMsgHandle(pWrapper);
   pWrapper->name = "snode";
   pWrapper->fp = mgmtFp;
 }

source/dnode/mgmt/test/vnode/vnode.cpp

@@ -70,46 +70,6 @@ TEST_F(DndTestVnode, 01_Create_Vnode) {
       ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED);
     }
   }
-
-  {
-    SCreateVnodeReq createReq = {0};
-    createReq.vgId = 2;
-    createReq.dnodeId = 3;
-    strcpy(createReq.db, "1.d1");
-    createReq.dbUid = 9527;
-    createReq.vgVersion = 1;
-    createReq.cacheBlockSize = 16;
-    createReq.totalBlocks = 10;
-    createReq.daysPerFile = 10;
-    createReq.daysToKeep0 = 3650;
-    createReq.daysToKeep1 = 3650;
-    createReq.daysToKeep2 = 3650;
-    createReq.minRows = 100;
-    createReq.minRows = 4096;
-    createReq.commitTime = 3600;
-    createReq.fsyncPeriod = 3000;
-    createReq.walLevel = 1;
-    createReq.precision = 0;
-    createReq.compression = 2;
-    createReq.replica = 1;
-    createReq.quorum = 1;
-    createReq.update = 0;
-    createReq.cacheLastRow = 0;
-    createReq.selfIndex = 0;
-    for (int r = 0; r < createReq.replica; ++r) {
-      SReplica* pReplica = &createReq.replicas[r];
-      pReplica->id = 1;
-      pReplica->port = 9527;
-    }
-
-    int32_t contLen = tSerializeSCreateVnodeReq(NULL, 0, &createReq);
-    void*   pReq = rpcMallocCont(contLen);
-    tSerializeSCreateVnodeReq(pReq, contLen, &createReq);
-
-    SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_VNODE, pReq, contLen);
-    ASSERT_NE(pRsp, nullptr);
-    ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_INVALID_OPTION);
-  }
 }
 
 TEST_F(DndTestVnode, 02_Alter_Vnode) {
@@ -164,37 +124,37 @@ TEST_F(DndTestVnode, 03_Create_Stb) {
     req.keep = 0;
     req.type = TD_SUPER_TABLE;
 
-    SSchema schemas[5] = {0};
+    SSchemaEx schemas[2] = {0};
     {
-      SSchema* pSchema = &schemas[0];
+      SSchemaEx* pSchema = &schemas[0];
       pSchema->bytes = htonl(8);
       pSchema->type = TSDB_DATA_TYPE_TIMESTAMP;
       strcpy(pSchema->name, "ts");
     }
 
     {
-      SSchema* pSchema = &schemas[1];
+      SSchemaEx* pSchema = &schemas[1];
       pSchema->bytes = htonl(4);
       pSchema->type = TSDB_DATA_TYPE_INT;
       strcpy(pSchema->name, "col1");
     }
-
+    SSchema tagSchemas[3] = {0};
     {
-      SSchema* pSchema = &schemas[2];
+      SSchema* pSchema = &tagSchemas[0];
       pSchema->bytes = htonl(2);
       pSchema->type = TSDB_DATA_TYPE_TINYINT;
       strcpy(pSchema->name, "tag1");
     }
 
     {
-      SSchema* pSchema = &schemas[3];
+      SSchema* pSchema = &tagSchemas[1];
       pSchema->bytes = htonl(8);
       pSchema->type = TSDB_DATA_TYPE_BIGINT;
       strcpy(pSchema->name, "tag2");
     }
 
     {
-      SSchema* pSchema = &schemas[4];
+      SSchema* pSchema = &tagSchemas[2];
       pSchema->bytes = htonl(16);
       pSchema->type = TSDB_DATA_TYPE_BINARY;
       strcpy(pSchema->name, "tag3");
@@ -204,7 +164,7 @@ TEST_F(DndTestVnode, 03_Create_Stb) {
     req.stbCfg.nCols = 2;
     req.stbCfg.pSchema = &schemas[0];
     req.stbCfg.nTagCols = 3;
-    req.stbCfg.pTagSchema = &schemas[2];
+    req.stbCfg.pTagSchema = &tagSchemas[0];
 
     int32_t   contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead);
     SMsgHead* pHead = (SMsgHead*)rpcMallocCont(contLen);
@@ -236,37 +196,37 @@ TEST_F(DndTestVnode, 04_Alter_Stb) {
     req.keep = 0;
     req.type = TD_SUPER_TABLE;
 
-    SSchema schemas[5] = {0};
+    SSchemaEx schemas[2] = {0};
     {
-      SSchema* pSchema = &schemas[0];
+      SSchemaEx* pSchema = &schemas[0];
       pSchema->bytes = htonl(8);
       pSchema->type = TSDB_DATA_TYPE_TIMESTAMP;
       strcpy(pSchema->name, "ts");
     }
 
     {
-      SSchema* pSchema = &schemas[1];
+      SSchemaEx* pSchema = &schemas[1];
       pSchema->bytes = htonl(4);
       pSchema->type = TSDB_DATA_TYPE_INT;
       strcpy(pSchema->name, "col1");
     }
-
+    SSchema tagSchemas[3] = {0};
     {
-      SSchema* pSchema = &schemas[2];
+      SSchema* pSchema = &tagSchemas[0];
       pSchema->bytes = htonl(2);
       pSchema->type = TSDB_DATA_TYPE_TINYINT;
       strcpy(pSchema->name, "_tag1");
     }
 
     {
-      SSchema* pSchema = &schemas[3];
+      SSchema* pSchema = &tagSchemas[1];
       pSchema->bytes = htonl(8);
       pSchema->type = TSDB_DATA_TYPE_BIGINT;
       strcpy(pSchema->name, "_tag2");
     }
 
     {
-      SSchema* pSchema = &schemas[4];
+      SSchema* pSchema = &tagSchemas[2];
       pSchema->bytes = htonl(16);
       pSchema->type = TSDB_DATA_TYPE_BINARY;
       strcpy(pSchema->name, "_tag3");
@@ -276,7 +236,7 @@ TEST_F(DndTestVnode, 04_Alter_Stb) {
     req.stbCfg.nCols = 2;
     req.stbCfg.pSchema = &schemas[0];
     req.stbCfg.nTagCols = 3;
-    req.stbCfg.pTagSchema = &schemas[2];
+    req.stbCfg.pTagSchema = &tagSchemas[0];
 
     int32_t   contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead);
     SMsgHead* pHead = (SMsgHead*)rpcMallocCont(contLen);

source/dnode/mgmt/vm/inc/vm.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_DND_VNODES_H_
-#define _TD_DND_VNODES_H_
-
-#include "dnd.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct {
-  int32_t openVnodes;
-  int32_t totalVnodes;
-  int32_t masterNum;
-  int64_t numOfSelectReqs;
-  int64_t numOfInsertReqs;
-  int64_t numOfInsertSuccessReqs;
-  int64_t numOfBatchInsertReqs;
-  int64_t numOfBatchInsertSuccessReqs;
-} SVnodesStat;
-
-void vmGetMgmtFp(SMgmtWrapper *pWrapper);
-
-void    vmMonitorVnodeLoads(SMgmtWrapper *pWrapper, SArray *pLoads);
-int32_t vmMonitorTfsInfo(SMgmtWrapper *pWrapper, SMonDiskInfo *pInfo);
-void    vmMonitorVnodeReqs(SMgmtWrapper *pWrapper, SMonDnodeInfo *pInfo);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_DND_VNODES_H_*/
\ No newline at end of file

source/dnode/mgmt/vm/src/vmMsg.c → source/dnode/mgmt/vm/vmHandle.c

@@ -30,6 +30,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
   pCfg->tsdbCfg.keep1 = pCreate->daysToKeep2;
   pCfg->tsdbCfg.keep2 = pCreate->daysToKeep0;
   pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize;
+  pCfg->tsdbCfg.retentions = pCreate->pRetensions;
   pCfg->metaCfg.lruSize = pCreate->cacheBlockSize;
   pCfg->walCfg.fsyncPeriod = pCreate->fsyncPeriod;
   pCfg->walCfg.level = pCreate->walLevel;
@@ -68,42 +69,35 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
   SWrapperCfg wrapperCfg = {0};
   vmGenerateWrapperCfg(pMgmt, &createReq, &wrapperCfg);
 
-  if (createReq.dnodeId != pMgmt->pDnode->dnodeId) {
-    terrno = TSDB_CODE_DND_VNODE_INVALID_OPTION;
-    dDebug("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr());
-    return -1;
-  }
-
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, createReq.vgId);
   if (pVnode != NULL) {
+    tFreeSCreateVnodeReq(&createReq);
     dDebug("vgId:%d, already exist", createReq.vgId);
     vmReleaseVnode(pMgmt, pVnode);
     terrno = TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED;
     return -1;
   }
 
-  SMsgCb msgCb = {0};
+  SMsgCb msgCb = dndCreateMsgcb(pMgmt->pWrapper);
   msgCb.pWrapper = pMgmt->pWrapper;
   msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue;
   msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue;
   msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue;
   msgCb.qsizeFp = vmGetQueueSize;
-  msgCb.sendReqFp = dndSendReqToDnode;
-  msgCb.sendMnodeReqFp = dndSendReqToMnode;
-  msgCb.sendRspFp = dndSendRsp;
-  msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
 
   vnodeCfg.msgCb = msgCb;
   vnodeCfg.pTfs = pMgmt->pTfs;
   vnodeCfg.dbId = wrapperCfg.dbUid;
   SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg);
   if (pImpl == NULL) {
+    tFreeSCreateVnodeReq(&createReq);
     dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr());
     return -1;
   }
 
   int32_t code = vmOpenVnode(pMgmt, &wrapperCfg, pImpl);
   if (code != 0) {
+    tFreeSCreateVnodeReq(&createReq);
     dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr());
     vnodeClose(pImpl);
     vnodeDestroy(wrapperCfg.path);
@@ -113,6 +107,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
 
   code = vmWriteVnodesToFile(pMgmt);
   if (code != 0) {
+    tFreeSCreateVnodeReq(&createReq);
     vnodeClose(pImpl);
     vnodeDestroy(wrapperCfg.path);
     terrno = code;
@@ -243,51 +238,51 @@ int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
   return 0;
 }
 
-void vmInitMsgHandles(SMgmtWrapper *pWrapper) {
+void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
   // Requests handled by VNODE
-  dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)vmProcessQueryMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, (NodeMsgFp)vmProcessQueryMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_TASK_PIPE_EXEC, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, (NodeMsgFp)vmProcessMergeMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_TASK_WRITE_EXEC, (NodeMsgFp)vmProcessWriteMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, (NodeMsgFp)vmProcessFetchMsg, VND_VGID);
-
-  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, vmProcessMgmtMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, VND_VGID);
-  dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, VND_VGID);
+  dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_TASK_PIPE_EXEC, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_TASK_MERGE_EXEC, (NodeMsgFp)vmProcessMergeMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_TASK_WRITE_EXEC, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_VND_STREAM_TRIGGER, (NodeMsgFp)vmProcessFetchMsg, DEFAULT_HANDLE);
+
+  dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
+  dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
 }

source/dnode/mgmt/vm/src/vmInt.c → source/dnode/mgmt/vm/vmInt.c

@@ -128,16 +128,12 @@ static void *vmOpenVnodeFunc(void *param) {
              pMgmt->state.openVnodes, pMgmt->state.totalVnodes);
     dndReportStartup(pDnode, "open-vnodes", stepDesc);
 
-    SMsgCb msgCb = {0};
+    SMsgCb msgCb = dndCreateMsgcb(pMgmt->pWrapper);
     msgCb.pWrapper = pMgmt->pWrapper;
     msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue;
     msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue;
     msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue;
     msgCb.qsizeFp = vmGetQueueSize;
-    msgCb.sendReqFp = dndSendReqToDnode;
-    msgCb.sendMnodeReqFp = dndSendReqToMnode;
-    msgCb.sendRspFp = dndSendRsp;
-    msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
     SVnodeCfg cfg = {.msgCb = msgCb, .pTfs = pMgmt->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid};
     SVnode   *pImpl = vnodeOpen(pCfg->path, &cfg);
     if (pImpl == NULL) {
@@ -337,13 +333,13 @@ static int32_t vmRequire(SMgmtWrapper *pWrapper, bool *required) {
   return 0;
 }
 
-void vmGetMgmtFp(SMgmtWrapper *pWrapper) {
+void vmSetMgmtFp(SMgmtWrapper *pWrapper) {
   SMgmtFp mgmtFp = {0};
   mgmtFp.openFp = vmInit;
   mgmtFp.closeFp = vmCleanup;
   mgmtFp.requiredFp = vmRequire;
 
-  vmInitMsgHandles(pWrapper);
+  vmInitMsgHandle(pWrapper);
   pWrapper->name = "vnode";
   pWrapper->fp = mgmtFp;
 }

source/dnode/mgmt/vm/src/vmWorker.c → source/dnode/mgmt/vm/vmWorker.c

@@ -18,7 +18,7 @@
 
 static void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
   SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code};
-  dndSendRsp(pWrapper, &rsp);
+  tmsgSendRsp(&rsp);
 }
 
 static void vmProcessMgmtQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
@@ -116,7 +116,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
     int32_t code = vnodeApplyWMsg(pVnode->pImpl, pRpc, &pRsp);
     if (pRsp != NULL) {
       pRsp->ahandle = pRpc->ahandle;
-      dndSendRsp(pVnode->pWrapper, pRsp);
+      tmsgSendRsp(pRsp);
       taosMemoryFree(pRsp);
     } else {
       if (code != 0 && terrno != 0) code = terrno;
@@ -187,7 +187,7 @@ static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueTyp
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
   if (pVnode == NULL) {
     dError("vgId:%d, failed to write msg:%p to vnode-queue since %s", pHead->vgId, pMsg, terrstr());
-    return -1;
+    return terrno;
   }
 
   int32_t code = 0;

source/dnode/mnode/impl/inc/mndCluster.h

@@ -25,6 +25,7 @@ extern "C" {
 int32_t mndInitCluster(SMnode *pMnode);
 void    mndCleanupCluster(SMnode *pMnode);
 int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len);
+int64_t mndGetClusterId(SMnode *pMnode);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/inc/mndDef.h

@@ -620,13 +620,13 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub)
 
 static FORCE_INLINE void tDeleteSMqSubscribeObj(SMqSubscribeObj* pSub) {
   if (pSub->consumers) {
-    taosArrayDestroyEx(pSub->consumers, (void (*)(void*))tDeleteSMqSubConsumer);
+    //taosArrayDestroyEx(pSub->consumers, (void (*)(void*))tDeleteSMqSubConsumer);
     // taosArrayDestroy(pSub->consumers);
     pSub->consumers = NULL;
   }
 
   if (pSub->unassignedVg) {
-    taosArrayDestroyEx(pSub->unassignedVg, (void (*)(void*))tDeleteSMqConsumerEp);
+    //taosArrayDestroyEx(pSub->unassignedVg, (void (*)(void*))tDeleteSMqConsumerEp);
     // taosArrayDestroy(pSub->unassignedVg);
     pSub->unassignedVg = NULL;
   }

source/dnode/mnode/impl/inc/mndInt.h

@@ -100,7 +100,6 @@ typedef struct {
 } SGrantInfo;
 
 typedef struct SMnode {
-  int32_t           dnodeId;
   int64_t           clusterId;
   int8_t            replica;
   int8_t            selfIndex;

source/dnode/mnode/impl/src/mndCluster.c

@@ -17,7 +17,7 @@
 #include "mndCluster.h"
 #include "mndShow.h"
 
-#define TSDB_CLUSTER_VER_NUMBE 1
+#define TSDB_CLUSTER_VER_NUMBE    1
 #define TSDB_CLUSTER_RESERVE_SIZE 64
 
 static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster);
@@ -61,6 +61,23 @@ int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) {
   return 0;
 }
 
+int64_t mndGetClusterId(SMnode *pMnode) {
+  SSdb   *pSdb = pMnode->pSdb;
+  void   *pIter = NULL;
+  int64_t clusterId = -1;
+
+  while (1) {
+    SClusterObj *pCluster = NULL;
+    pIter = sdbFetch(pSdb, SDB_CLUSTER, pIter, (void **)&pCluster);
+    if (pIter == NULL) break;
+
+    clusterId = pCluster->id;
+    sdbRelease(pSdb, pCluster);
+  }
+
+  return clusterId;
+}
+
 static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) {
   terrno = TSDB_CODE_OUT_OF_MEMORY;
 

source/dnode/mnode/impl/src/mndConsumer.c

@@ -160,9 +160,8 @@ static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) {
 
 static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) {
   mTrace("consumer:%" PRId64 ", perform update action", pOldConsumer->consumerId);
-
-  // TODO handle update
   /*taosWLockLatch(&pOldConsumer->lock);*/
+  atomic_add_fetch_32(&pOldConsumer->epoch, 1);
   /*taosWUnLockLatch(&pOldConsumer->lock);*/
 
   return 0;

source/dnode/mnode/impl/src/mndDb.c

@@ -69,7 +69,8 @@ void mndCleanupDb(SMnode *pMnode) {}
 static SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
   terrno = TSDB_CODE_OUT_OF_MEMORY;
 
-  SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUMBER, sizeof(SDbObj) + TSDB_DB_RESERVE_SIZE);
+  SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUMBER,
+                              sizeof(SDbObj) + pDb->cfg.numOfRetensions * sizeof(SRetention) + TSDB_DB_RESERVE_SIZE);
   if (pRaw == NULL) goto DB_ENCODE_OVER;
 
   int32_t dataPos = 0;
@@ -1309,7 +1310,7 @@ static int32_t mndGetDbMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMet
   cols++;
 
   pShow->bytes[cols] = 2;
-  pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "days");
   pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
@@ -1444,7 +1445,7 @@ static void dumpDbInfoToPayload(char *data, SDbObj *pDb, SShowObj *pShow, int32_
   cols++;
 
   pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
-  *(int16_t *)pWrite = pDb->cfg.daysPerFile;
+  *(int32_t *)pWrite = pDb->cfg.daysPerFile;
   cols++;
 
   pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);

source/dnode/mnode/impl/src/mndInfoSchema.c

@@ -53,7 +53,7 @@ static const SInfosTableSchema userDBSchema[] = {
     {.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
     {.name = "replica", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
     {.name = "quorum", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
-    {.name = "days", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
+    {.name = "days", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "keep", .bytes = 24 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "cache", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT},

source/dnode/mnode/impl/src/mndProfile.c

@@ -433,12 +433,6 @@ static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) {
   pHeartbeat->connId = htonl(pHeartbeat->connId);
   pHeartbeat->pid = htonl(pHeartbeat->pid);
 
-  SRpcConnInfo info = {0};
-  if (rpcGetConnInfo(pReq->rpcMsg.handle, &info) != 0) {
-    mError("user:%s, connId:%d failed to process hb since %s", pReq->user, pHeartbeat->connId, terrstr());
-    return -1;
-  }
-
   SConnObj *pConn = mndAcquireConn(pMnode, pHeartbeat->connId);
   if (pConn == NULL) {
     pConn = mndCreateConn(pMnode, &info, pHeartbeat->pid, pHeartbeat->app, 0);

source/dnode/mnode/impl/src/mndQnode.c

@@ -423,7 +423,7 @@ static int32_t mndProcessDropQnodeReq(SNodeMsg *pReq) {
 
 DROP_QNODE_OVER:
   if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
-    mError("qnode:%d, failed to drop since %s", pMnode->dnodeId, terrstr());
+    mError("qnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr());
   }
 
   mndReleaseQnode(pMnode, pObj);

source/dnode/mnode/impl/src/mndQuery.c

@@ -52,7 +52,7 @@ int32_t mndProcessFetchMsg(SNodeMsg *pReq) {
 }
 
 int32_t mndInitQuery(SMnode *pMnode) {
-  if (qWorkerInit(NODE_TYPE_MNODE, MND_VGID, NULL, (void **)&pMnode->pQuery, &pMnode->msgCb) != 0) {
+  if (qWorkerInit(NODE_TYPE_MNODE, MNODE_HANDLE, NULL, (void **)&pMnode->pQuery, &pMnode->msgCb) != 0) {
     mError("failed to init qworker in mnode since %s", terrstr());
     return -1;
   }

source/dnode/mnode/impl/src/mndScheduler.c

@@ -471,6 +471,9 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     consumerEp.consumerId = -1;
     consumerEp.epSet = plan->execNode.epSet;
     consumerEp.vgId = plan->execNode.nodeId;
+    
+    mDebug("init subscribption %s, assign vg: %d", pSub->key, consumerEp.vgId);
+
     int32_t msgLen;
     if (qSubPlanToString(plan, &consumerEp.qmsg, &msgLen) < 0) {
       sdbRelease(pSdb, pVgroup);

source/dnode/mnode/impl/src/mndSnode.c

@@ -433,7 +433,7 @@ static int32_t mndProcessDropSnodeReq(SNodeMsg *pReq) {
 
 DROP_SNODE_OVER:
   if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
-    mError("snode:%d, failed to drop since %s", pMnode->dnodeId, terrstr());
+    mError("snode:%d, failed to drop since %s", dropReq.dnodeId, terrstr());
   }
 
   mndReleaseSnode(pMnode, pObj);

source/dnode/mnode/impl/test/sma/sma.cpp

@@ -42,10 +42,10 @@ void* MndTestSma::BuildCreateDbReq(const char* dbname, int32_t* pContLen) {
   createReq.numOfVgroups = 2;
   createReq.cacheBlockSize = 16;
   createReq.totalBlocks = 10;
-  createReq.daysPerFile = 10;
-  createReq.daysToKeep0 = 3650;
-  createReq.daysToKeep1 = 3650;
-  createReq.daysToKeep2 = 3650;
+  createReq.daysPerFile = 10 * 1440;
+  createReq.daysToKeep0 = 3650 * 1440;
+  createReq.daysToKeep1 = 3650 * 1440;
+  createReq.daysToKeep2 = 3650 * 1440;
   createReq.minRows = 100;
   createReq.maxRows = 4096;
   createReq.commitTime = 3600;

source/dnode/mnode/impl/test/topic/topic.cpp

@@ -35,10 +35,10 @@ void* MndTestTopic::BuildCreateDbReq(const char* dbname, int32_t* pContLen) {
   createReq.numOfVgroups = 2;
   createReq.cacheBlockSize = 16;
   createReq.totalBlocks = 10;
-  createReq.daysPerFile = 10;
-  createReq.daysToKeep0 = 3650;
-  createReq.daysToKeep1 = 3650;
-  createReq.daysToKeep2 = 3650;
+  createReq.daysPerFile = 10 * 1440;
+  createReq.daysToKeep0 = 3650 * 1440;
+  createReq.daysToKeep1 = 3650 * 1440;
+  createReq.daysToKeep2 = 3650 * 1440;
   createReq.minRows = 100;
   createReq.maxRows = 4096;
   createReq.commitTime = 3600;