Merge branch '3.0' into feature/3.0_glzhao

.clang-format

@@ -5,6 +5,7 @@ AccessModifierOffset: -1
 AlignAfterOpenBracket: Align
 AlignConsecutiveAssignments: false
 AlignConsecutiveDeclarations: true
+AlignConsecutiveMacros: true
 AlignEscapedNewlinesLeft: true
 AlignOperands:   true
 AlignTrailingComments: true

example/src/tmq.c

@@ -28,7 +28,7 @@ int32_t init_env() {
     return -1;
   }
 
-  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1");
+  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2");
   if (taos_errno(pRes) != 0) {
     printf("error in create db, reason:%s\n", taos_errstr(pRes));
     return -1;
@@ -42,25 +42,33 @@ int32_t init_env() {
   }
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int) tags(a int)");
+  pRes =
+      taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)");
   if (taos_errno(pRes) != 0) {
     printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes));
     return -1;
   }
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "create table if not exists tu1 using st1 tags(1)");
+  pRes = taos_query(pConn, "create table if not exists ct0 using st1 tags(1000)");
   if (taos_errno(pRes) != 0) {
     printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes));
     return -1;
   }
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "create table if not exists tu2 using st1 tags(2)");
+  pRes = taos_query(pConn, "create table if not exists ct1 using st1 tags(2000)");
   if (taos_errno(pRes) != 0) {
     printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes));
     return -1;
   }
+
+  pRes = taos_query(pConn, "create table if not exists ct3 using st1 tags(3000)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create child table tu3, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+
   taos_free_result(pRes);
   return 0;
 }
@@ -82,12 +90,40 @@ int32_t create_topic() {
 
   /*const char* sql = "select * from tu1";*/
   /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));*/
-  pRes = taos_query(pConn, "create topic test_stb_topic_1 as select * from tu1");
+  pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1 from ct1");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+#if 0
+  pRes = taos_query(pConn, "insert into tu1 values(now, 1, 1.0, 'bi1')");
   if (taos_errno(pRes) != 0) {
-    printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes));
+    printf("failed to insert, reason:%s\n", taos_errstr(pRes));
     return -1;
   }
   taos_free_result(pRes);
+  pRes = taos_query(pConn, "insert into tu1 values(now+1d, 1, 1.0, 'bi1')");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to insert, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+  pRes = taos_query(pConn, "insert into tu2 values(now, 2, 2.0, 'bi2')");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to insert, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+  pRes = taos_query(pConn, "insert into tu2 values(now+1d, 2, 2.0, 'bi2')");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to insert, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+#endif
+
   taos_close(pConn);
   return 0;
 }
@@ -115,7 +151,7 @@ tmq_t* build_consumer() {
 
 tmq_list_t* build_topic_list() {
   tmq_list_t* topic_list = tmq_list_new();
-  tmq_list_append(topic_list, "test_stb_topic_1");
+  tmq_list_append(topic_list, "topic_ctb_column");
   return topic_list;
 }
 
@@ -215,8 +251,8 @@ int main(int argc, char* argv[]) {
   if (argc > 1) {
     printf("env init\n");
     code = init_env();
+    create_topic();
   }
-  create_topic();
   tmq_t*      tmq = build_consumer();
   tmq_list_t* topic_list = build_topic_list();
   /*perf_loop(tmq, topic_list);*/

example/src/tstream.c

@@ -20,7 +20,7 @@
 #include "taos.h"
 
 int32_t init_env() {
-  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 7010);
+  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   if (pConn == NULL) {
     return -1;
   }
@@ -65,7 +65,7 @@ int32_t init_env() {
 int32_t create_stream() {
   printf("create stream\n");
   TAOS_RES* pRes;
-  TAOS*     pConn = taos_connect("localhost", "root", "taosdata", NULL, 7010);
+  TAOS*     pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   if (pConn == NULL) {
     return -1;
   }

include/client/taos.h

@@ -54,6 +54,7 @@ typedef void   TAOS_SUB;
 #define TSDB_DATA_TYPE_BLOB       18  // binary
 #define TSDB_DATA_TYPE_MEDIUMBLOB 19
 #define TSDB_DATA_TYPE_BINARY     TSDB_DATA_TYPE_VARCHAR  // string
+#define TSDB_DATA_TYPE_MAX        20
 
 typedef enum {
   TSDB_OPTION_LOCALE,
@@ -124,8 +125,25 @@ typedef struct TAOS_MULTI_BIND {
   int       num;
 } TAOS_MULTI_BIND;
 
+typedef enum {
+  SET_CONF_RET_SUCC = 0,
+  SET_CONF_RET_ERR_PART = -1,
+  SET_CONF_RET_ERR_INNER = -2,
+  SET_CONF_RET_ERR_JSON_INVALID = -3,
+  SET_CONF_RET_ERR_JSON_PARSE = -4,
+  SET_CONF_RET_ERR_ONLY_ONCE = -5,
+  SET_CONF_RET_ERR_TOO_LONG = -6
+} SET_CONF_RET_CODE;
+
+#define RET_MSG_LENGTH 1024
+typedef struct setConfRet {
+  SET_CONF_RET_CODE retCode;
+  char   retMsg[RET_MSG_LENGTH];
+} setConfRet;
+
 DLL_EXPORT void  taos_cleanup(void);
 DLL_EXPORT int   taos_options(TSDB_OPTION option, const void *arg, ...);
+DLL_EXPORT setConfRet   taos_set_config(const char *config);
 DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
 DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen,
                                 const char *db, int dbLen, uint16_t port);
@@ -168,10 +186,13 @@ DLL_EXPORT int         taos_select_db(TAOS *taos, const char *db);
 DLL_EXPORT int         taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields);
 DLL_EXPORT void        taos_stop_query(TAOS_RES *res);
 DLL_EXPORT bool        taos_is_null(TAOS_RES *res, int32_t row, int32_t col);
+DLL_EXPORT bool        taos_is_update_query(TAOS_RES *res);
 DLL_EXPORT int         taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows);
 DLL_EXPORT int         taos_validate_sql(TAOS *taos, const char *sql);
+DLL_EXPORT void        taos_reset_current_db(TAOS *taos);
 
 DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res);
+DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res);
 
 DLL_EXPORT const char *taos_get_server_info(TAOS *taos);
 DLL_EXPORT const char *taos_get_client_info();

include/common/taosdef.h

@@ -76,6 +76,13 @@ typedef enum {
   TSDB_SMA_TYPE_ROLLUP = 2,      // Rollup SMA
 } ETsdbSmaType;
 
+typedef enum {
+  TSDB_BSMA_TYPE_NONE = 0,  // no block-wise SMA
+  TSDB_BSMA_TYPE_I = 1,     // sum/min/max(default)
+} ETsdbBSmaType;
+
+#define TSDB_BSMA_TYPE_LATEST TSDB_BSMA_TYPE_I
+
 extern char *qtypeStr[];
 
 #define TSDB_PORT_HTTP      11

include/common/tcommon.h

@@ -37,6 +37,14 @@ enum {
   TMQ_MSG_TYPE__EP_RSP,
 };
 
+enum {
+  STREAM_TRIGGER__AT_ONCE = 1,
+  STREAM_TRIGGER__WINDOW_CLOSE,
+  STREAM_TRIGGER__BY_COUNT,
+  STREAM_TRIGGER__BY_BATCH_COUNT,
+  STREAM_TRIGGER__BY_EVENT_TIME,
+};
+
 typedef struct {
   uint32_t  numOfTables;
   SArray*   pGroupList;
@@ -54,13 +62,16 @@ typedef struct SColumnDataAgg {
 } SColumnDataAgg;
 
 typedef struct SDataBlockInfo {
-  STimeWindow    window;
-  int32_t        rows;
-  int32_t        rowSize;
-  int16_t        numOfCols;
-  int16_t        hasVarCol;
-  union {int64_t uid; int64_t blockId;};
-  int64_t        groupId;     // no need to serialize
+  STimeWindow window;
+  int32_t     rows;
+  int32_t     rowSize;
+  int16_t     numOfCols;
+  int16_t     hasVarCol;
+  union {
+    int64_t uid;
+    int64_t blockId;
+  };
+  int64_t groupId;  // no need to serialize
 } SDataBlockInfo;
 
 typedef struct SSDataBlock {
@@ -93,7 +104,7 @@ void*   tDecodeDataBlock(const void* buf, SSDataBlock* pBlock);
 
 int32_t tEncodeDataBlocks(void** buf, const SArray* blocks);
 void*   tDecodeDataBlocks(const void* buf, SArray** blocks);
-void    colDataDestroy(SColumnInfoData* pColData) ;
+void    colDataDestroy(SColumnInfoData* pColData);
 
 static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) {
   // WARNING: do not use info.numOfCols,
@@ -198,8 +209,8 @@ typedef struct SGroupbyExpr {
 } SGroupbyExpr;
 
 enum {
-  FUNC_PARAM_TYPE_VALUE = 0,
-  FUNC_PARAM_TYPE_COLUMN,
+  FUNC_PARAM_TYPE_VALUE = 0x1,
+  FUNC_PARAM_TYPE_COLUMN= 0x2,
 };
 
 typedef struct SFunctParam {
@@ -239,7 +250,7 @@ typedef struct SSessionWindow {
   SColumn col;
 } SSessionWindow;
 
-#define QUERY_ASC_FORWARD_STEP  1
+#define QUERY_ASC_FORWARD_STEP 1
 #define QUERY_DESC_FORWARD_STEP -1
 
 #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP)

include/common/tdatablock.h

@@ -133,7 +133,8 @@ static FORCE_INLINE int32_t colDataAppendInt32(SColumnInfoData* pColumnInfoData,
 }
 
 static FORCE_INLINE int32_t colDataAppendInt64(SColumnInfoData* pColumnInfoData, uint32_t currentRow, int64_t* v) {
-  ASSERT(pColumnInfoData->info.type == TSDB_DATA_TYPE_BIGINT || pColumnInfoData->info.type == TSDB_DATA_TYPE_UBIGINT);
+  int32_t type = pColumnInfoData->info.type;
+  ASSERT(type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_UBIGINT || type == TSDB_DATA_TYPE_TIMESTAMP);
   char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow;
   *(int64_t*)p = *(int64_t*)v;
 }
@@ -175,17 +176,18 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock);
 double blockDataGetSerialRowSize(const SSDataBlock* pBlock);
 size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock);
 
-SSchema* blockDataExtractSchema(const SSDataBlock* pBlock, int32_t* numOfCols);
-
 int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo);
 int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
 
-int32_t      blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows);
-int32_t      blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
-void         blockDataCleanup(SSDataBlock* pDataBlock);
+int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows);
+int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
+void    blockDataCleanup(SSDataBlock* pDataBlock);
+size_t  blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize);
+void*   blockDataDestroy(SSDataBlock* pBlock);
+
+int32_t blockDataTrimFirstNRows(SSDataBlock *pBlock, size_t n);
+
 SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock);
-size_t       blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize);
-void*        blockDataDestroy(SSDataBlock* pBlock);
 
 void blockDebugShowData(const SArray* dataBlocks);
 

include/common/tdataformat.h

@@ -70,11 +70,13 @@ typedef struct {
 #pragma pack(pop)
 
 #define colType(col)   ((col)->type)
+#define colSma(col)    ((col)->sma)
 #define colColId(col)  ((col)->colId)
 #define colBytes(col)  ((col)->bytes)
 #define colOffset(col) ((col)->offset)
 
 #define colSetType(col, t)   (colType(col) = (t))
+#define colSetSma(col, s)    (colSma(col) = (s))
 #define colSetColId(col, id) (colColId(col) = (id))
 #define colSetBytes(col, b)  (colBytes(col) = (b))
 #define colSetOffset(col, o) (colOffset(col) = (o))
@@ -139,7 +141,7 @@ typedef struct {
 int32_t   tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
 void      tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder);
 void      tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
-int32_t   tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col_bytes_t bytes);
+int32_t   tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col_id_t colId, col_bytes_t bytes);
 STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
 
 // ----------------- Semantic timestamp key definition

include/common/tglobal.h

@@ -51,7 +51,7 @@ extern int32_t tsCompatibleModel;
 extern bool    tsEnableSlaveQuery;
 extern bool    tsPrintAuth;
 extern int64_t tsTickPerDay[3];
-extern int32_t tsMultiProcess;
+extern bool    tsMultiProcess;
 
 // monitor
 extern bool     tsEnableMonitor;

include/common/tmsg.h

@@ -265,6 +265,20 @@ typedef struct SSchema {
   char     name[TSDB_COL_NAME_LEN];
 } SSchema;
 
+typedef struct {
+  int8_t   type;
+  int8_t   sma;  // ETsdbBSmaType and default is TSDB_BSMA_TYPE_I
+  col_id_t colId;
+  int32_t  bytes;
+  char     name[TSDB_COL_NAME_LEN];
+} SSchemaEx;
+
+#define SSCHMEA_TYPE(s) ((s)->type)
+#define SSCHMEA_SMA(s) ((s)->sma)
+#define SSCHMEA_COLID(s) ((s)->colId)
+#define SSCHMEA_BYTES(s) ((s)->bytes)
+#define SSCHMEA_NAME(s) ((s)->name)
+
 typedef struct {
   char    name[TSDB_TABLE_FNAME_LEN];
   int8_t  igExists;
@@ -469,8 +483,7 @@ typedef struct {
   int32_t tz;  // query client timezone
   char    intervalUnit;
   char    slidingUnit;
-  char
-      offsetUnit;  // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration.
+  char    offsetUnit;  // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration.
   int8_t  precision;
   int64_t interval;
   int64_t sliding;
@@ -525,6 +538,7 @@ typedef struct {
   int8_t  walLevel;
   int8_t  quorum;
   int8_t  cacheLastRow;
+  int8_t  replications;
 } SAlterDbReq;
 
 int32_t tSerializeSAlterDbReq(void* buf, int32_t bufLen, SAlterDbReq* pReq);
@@ -1403,13 +1417,12 @@ typedef struct SVCreateTbReq {
   };
   union {
     struct {
-      tb_uid_t    suid;
-      uint32_t    nCols;
-      SSchema*    pSchema;
-      uint32_t    nTagCols;
-      SSchema*    pTagSchema;
-      col_id_t    nBSmaCols;
-      col_id_t*   pBSmaCols;
+      tb_uid_t   suid;
+      col_id_t   nCols;
+      col_id_t   nBSmaCols;
+      SSchemaEx* pSchema;
+      col_id_t   nTagCols;
+      SSchema*   pTagSchema;
       SRSmaParam* pRSmaParam;
     } stbCfg;
     struct {
@@ -1417,10 +1430,9 @@ typedef struct SVCreateTbReq {
       SKVRow   pTag;
     } ctbCfg;
     struct {
-      uint32_t    nCols;
-      SSchema*    pSchema;
-      col_id_t    nBSmaCols;
-      col_id_t*   pBSmaCols;
+      col_id_t   nCols;
+      col_id_t   nBSmaCols;
+      SSchemaEx* pSchema;
       SRSmaParam* pRSmaParam;
     } ntbCfg;
   };
@@ -1899,7 +1911,10 @@ int32_t tDecodeSMqCMCommitOffsetReq(SCoder* decoder, SMqCMCommitOffsetReq* pReq)
 
 typedef struct {
   uint32_t nCols;
-  SSchema* pSchema;
+  union {
+    SSchema*   pSchema;
+    SSchemaEx* pSchemaEx;
+  };
 } SSchemaWrapper;
 
 static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema) {
@@ -2017,7 +2032,6 @@ typedef struct {
   int8_t   slidingUnit;   // MACRO: TIME_UNIT_XXX
   int8_t   timezoneInt;   // sma data expired if timezone changes.
   char     indexName[TSDB_INDEX_NAME_LEN];
-  char     timezone[TD_TIMEZONE_LEN];
   int32_t  exprLen;
   int32_t  tagsFilterLen;
   int64_t  indexUid;
@@ -2055,32 +2069,6 @@ void*   tDeserializeSVCreateTSmaReq(void* buf, SVCreateTSmaReq* pReq);
 int32_t tSerializeSVDropTSmaReq(void** buf, SVDropTSmaReq* pReq);
 void*   tDeserializeSVDropTSmaReq(void* buf, SVDropTSmaReq* pReq);
 
-typedef struct {
-  col_id_t colId;
-  uint16_t blockSize;  // sma data block size
-  char     data[];
-} STSmaColData;
-
-typedef struct {
-  tb_uid_t tableUid;  // super/child/normal table uid
-  int32_t  dataLen;   // not including head
-  char     data[];
-} STSmaTbData;
-
-typedef struct {
-  int64_t indexUid;
-  TSKEY   skey;  // startKey of one interval/sliding window
-  int64_t interval;
-  int32_t dataLen;  // not including head
-  int8_t  intervalUnit;
-  char    data[];
-} STSmaDataWrapper;  // sma data for a interval/sliding window
-
-// interval/sliding => window
-
-// => window->table->colId
-// => 当一个window下所有的表均计算完成时，流计算告知tsdb清除window的过期标记
-
 // RSma: Rollup SMA
 typedef struct {
   int64_t  interval;

include/common/trow.h

@@ -47,21 +47,21 @@ extern "C" {
 #define TD_VTYPE_NONE 0x0U   // none or unknown/undefined
 #define TD_VTYPE_NULL 0x01U  // null val
 #define TD_VTYPE_NORM 0x02U  // normal val: not none, not null
-#define TD_VTYPE_MAX  0x03U  //
+#define TD_VTYPE_MAX 0x03U   //
 
 #define TD_VTYPE_NONE_BYTE 0x0U
 #define TD_VTYPE_NULL_BYTE 0x55U
 #define TD_VTYPE_NORM_BYTE 0xAAU
 
-#define TD_ROWS_ALL_NORM  0x01U
+#define TD_ROWS_ALL_NORM 0x01U
 #define TD_ROWS_NULL_NORM 0x0U
 
-#define TD_COL_ROWS_NORM(c)          ((c)->bitmap == TD_ROWS_ALL_NORM)  // all rows of SDataCol/SBlockCol is NORM
+#define TD_COL_ROWS_NORM(c) ((c)->bitmap == TD_ROWS_ALL_NORM)  // all rows of SDataCol/SBlockCol is NORM
 #define TD_SET_COL_ROWS_BTIMAP(c, v) ((c)->bitmap = (v))
-#define TD_SET_COL_ROWS_NORM(c)      TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_ALL_NORM)
-#define TD_SET_COL_ROWS_MISC(c)      TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_NULL_NORM)
+#define TD_SET_COL_ROWS_NORM(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_ALL_NORM)
+#define TD_SET_COL_ROWS_MISC(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_NULL_NORM)
 
-#define KvConvertRatio            (0.9f)
+#define KvConvertRatio (0.9f)
 #define isSelectKVRow(klen, tlen) ((klen) < ((tlen)*KvConvertRatio))
 
 #ifdef TD_SUPPORT_BITMAP
@@ -98,7 +98,7 @@ typedef void *SRow;
 
 typedef struct {
   TDRowValT valType;
-  void *    val;
+  void     *val;
 } SCellVal;
 
 typedef struct {
@@ -158,43 +158,43 @@ typedef struct {
   int16_t nBitmaps;
   int16_t nBoundBitmaps;
   int32_t offset;
-  void *  pBitmap;
-  void *  pOffset;
+  void   *pBitmap;
+  void   *pOffset;
   int32_t extendedRowSize;
 } SRowBuilder;
 
-#define TD_ROW_HEAD_LEN  (sizeof(STSRow))
+#define TD_ROW_HEAD_LEN (sizeof(STSRow))
 #define TD_ROW_NCOLS_LEN (sizeof(col_id_t))
 
-#define TD_ROW_INFO(r)     ((r)->info)
-#define TD_ROW_TYPE(r)     ((r)->type)
-#define TD_ROW_DELETE(r)   ((r)->del)
-#define TD_ROW_ENDIAN(r)   ((r)->endian)
-#define TD_ROW_SVER(r)     ((r)->sver)
-#define TD_ROW_NCOLS(r)    ((r)->data)  // only valid for SKvRow
-#define TD_ROW_DATA(r)     ((r)->data)
-#define TD_ROW_LEN(r)      ((r)->len)
-#define TD_ROW_KEY(r)      ((r)->ts)
+#define TD_ROW_INFO(r) ((r)->info)
+#define TD_ROW_TYPE(r) ((r)->type)
+#define TD_ROW_DELETE(r) ((r)->del)
+#define TD_ROW_ENDIAN(r) ((r)->endian)
+#define TD_ROW_SVER(r) ((r)->sver)
+#define TD_ROW_NCOLS(r) ((r)->data)  // only valid for SKvRow
+#define TD_ROW_DATA(r) ((r)->data)
+#define TD_ROW_LEN(r) ((r)->len)
+#define TD_ROW_KEY(r) ((r)->ts)
 #define TD_ROW_KEY_ADDR(r) (r)
 
 // N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and
 // (int32_t)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined.
 #define TD_ROW_MAX_BYTES_FROM_SCHEMA(s) (schemaTLen(s) + TD_ROW_HEAD_LEN)
 
-#define TD_ROW_SET_INFO(r, i)  (TD_ROW_INFO(r) = (i))
-#define TD_ROW_SET_TYPE(r, t)  (TD_ROW_TYPE(r) = (t))
-#define TD_ROW_SET_DELETE(r)   (TD_ROW_DELETE(r) = 1)
-#define TD_ROW_SET_SVER(r, v)  (TD_ROW_SVER(r) = (v))
-#define TD_ROW_SET_LEN(r, l)   (TD_ROW_LEN(r) = (l))
+#define TD_ROW_SET_INFO(r, i) (TD_ROW_INFO(r) = (i))
+#define TD_ROW_SET_TYPE(r, t) (TD_ROW_TYPE(r) = (t))
+#define TD_ROW_SET_DELETE(r) (TD_ROW_DELETE(r) = 1)
+#define TD_ROW_SET_SVER(r, v) (TD_ROW_SVER(r) = (v))
+#define TD_ROW_SET_LEN(r, l) (TD_ROW_LEN(r) = (l))
 #define TD_ROW_SET_NCOLS(r, n) (*(col_id_t *)TD_ROW_NCOLS(r) = (n))
 
 #define TD_ROW_IS_DELETED(r) (TD_ROW_DELETE(r) == 1)
-#define TD_IS_TP_ROW(r)      (TD_ROW_TYPE(r) == TD_ROW_TP)
-#define TD_IS_KV_ROW(r)      (TD_ROW_TYPE(r) == TD_ROW_KV)
-#define TD_IS_TP_ROW_T(t)    ((t) == TD_ROW_TP)
-#define TD_IS_KV_ROW_T(t)    ((t) == TD_ROW_KV)
+#define TD_IS_TP_ROW(r) (TD_ROW_TYPE(r) == TD_ROW_TP)
+#define TD_IS_KV_ROW(r) (TD_ROW_TYPE(r) == TD_ROW_KV)
+#define TD_IS_TP_ROW_T(t) ((t) == TD_ROW_TP)
+#define TD_IS_KV_ROW_T(t) ((t) == TD_ROW_KV)
 
-#define TD_BOOL_STR(b)       ((b) ? "true" : "false")
+#define TD_BOOL_STR(b) ((b) ? "true" : "false")
 #define isUtilizeKVRow(k, d) ((k) < ((d)*KVRatioConvert))
 
 #define TD_ROW_COL_IDX(r) POINTER_SHIFT(TD_ROW_DATA(r), sizeof(col_id_t))
@@ -275,7 +275,7 @@ static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TD
   }
   int16_t nBytes = colIdx / TD_VTYPE_PARTS;
   int16_t nOffset = colIdx & TD_VTYPE_OPTR;
-  char *  pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes);
+  char   *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes);
   switch (nOffset) {
     case 0:
       *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6);
@@ -313,7 +313,7 @@ static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TD
   }
   int16_t nBytes = colIdx / TD_VTYPE_PARTS;
   int16_t nOffset = colIdx & TD_VTYPE_OPTR;
-  char *  pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes);
+  char   *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes);
   switch (nOffset) {
     case 0:
       *pValType = (((*pDestByte) & 0xC0) >> 6);
@@ -620,7 +620,7 @@ static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowVa
   if (tdValIsNorm(valType, val, colType)) {
     // ts key stored in STSRow.ts
     SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset);
-    char *     ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row));
+    char      *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row));
     pColIdx->colId = colId;
     pColIdx->offset = TD_ROW_LEN(row);  // the offset include the TD_ROW_HEAD_LEN
 
@@ -638,7 +638,7 @@ static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowVa
   // NULL/None value
   else {
     SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset);
-    char *     ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row));
+    char      *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row));
     pColIdx->colId = colId;
     pColIdx->offset = TD_ROW_LEN(row);  // the offset include the TD_ROW_HEAD_LEN
     const void *nullVal = getNullValue(colType);
@@ -775,8 +775,8 @@ static FORCE_INLINE int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, v
 
 typedef struct {
   STSchema *pSchema;
-  STSRow *  pRow;
-  void *    pBitmap;
+  STSRow   *pRow;
+  void     *pBitmap;
   uint32_t  offset;
   col_id_t  maxColId;
   col_id_t  colIdx;  // [PRIMARYKEY_TIMESTAMP_COL_ID, nSchemaCols], PRIMARYKEY_TIMESTAMP_COL_ID equals 1
@@ -881,7 +881,7 @@ static FORCE_INLINE bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colTy
 // internal
 static FORCE_INLINE bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx,
                                               SCellVal *pVal) {
-  STSRow *   pRow = pIter->pRow;
+  STSRow    *pRow = pIter->pRow;
   SKvRowIdx *pKvIdx = NULL;
   bool       colFound = false;
   col_id_t   kvNCols = tdRowGetNCols(pRow);
@@ -937,9 +937,8 @@ static FORCE_INLINE bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col
     STColumn *pCol = NULL;
     STSchema *pSchema = pIter->pSchema;
     while (pIter->colIdx <= pSchema->numOfCols) {
-      pCol = &pSchema->columns[pIter->colIdx];
+      pCol = &pSchema->columns[pIter->colIdx];  // 1st column of schema is primary TS key
       if (colId == pCol->colId) {
-        ++pIter->colIdx;
         break;
       } else if (colId < pCol->colId) {
         ++pIter->colIdx;
@@ -948,7 +947,8 @@ static FORCE_INLINE bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col
         return false;
       }
     }
-    return tdGetTpRowDataOfCol(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal);
+    tdGetTpRowDataOfCol(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal);
+    ++pIter->colIdx;
   } else if (TD_IS_KV_ROW(pIter->pRow)) {
     return tdGetKvRowValOfColEx(pIter, colId, colType, &pIter->kvIdx, pVal);
   } else {
@@ -1076,7 +1076,7 @@ typedef struct {
 
 typedef struct {
   STSchema *pSchema;
-  STSRow *  pRow;
+  STSRow   *pRow;
 } STSRowReader;
 
 typedef struct {

include/common/ttime.h

@@ -60,8 +60,10 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati
 
 int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth);
 void    deltaToUtcInitOnce();
+char getPrecisionUnit(int32_t precision);
 
 int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision);
+int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit);
 
 void taosFormatUtcTime(char *buf, int32_t bufLen, int64_t time, int32_t precision);
 

include/common/ttokendef.h

@@ -82,21 +82,21 @@
 #define TK_SINGLE_STABLE                   64
 #define TK_STREAM_MODE                     65
 #define TK_RETENTIONS                      66
-#define TK_TABLE                           67
-#define TK_NK_LP                           68
-#define TK_NK_RP                           69
-#define TK_STABLE                          70
-#define TK_ADD                             71
-#define TK_COLUMN                          72
-#define TK_MODIFY                          73
-#define TK_RENAME                          74
-#define TK_TAG                             75
-#define TK_SET                             76
-#define TK_NK_EQ                           77
-#define TK_USING                           78
-#define TK_TAGS                            79
-#define TK_NK_DOT                          80
-#define TK_NK_COMMA                        81
+#define TK_NK_COMMA                        67
+#define TK_TABLE                           68
+#define TK_NK_LP                           69
+#define TK_NK_RP                           70
+#define TK_STABLE                          71
+#define TK_ADD                             72
+#define TK_COLUMN                          73
+#define TK_MODIFY                          74
+#define TK_RENAME                          75
+#define TK_TAG                             76
+#define TK_SET                             77
+#define TK_NK_EQ                           78
+#define TK_USING                           79
+#define TK_TAGS                            80
+#define TK_NK_DOT                          81
 #define TK_COMMENT                         82
 #define TK_BOOL                            83
 #define TK_TINYINT                         84
@@ -131,62 +131,91 @@
 #define TK_FUNCTIONS                      113
 #define TK_INDEXES                        114
 #define TK_FROM                           115
-#define TK_LIKE                           116
-#define TK_INDEX                          117
-#define TK_FULLTEXT                       118
-#define TK_FUNCTION                       119
-#define TK_INTERVAL                       120
-#define TK_TOPIC                          121
-#define TK_AS                             122
-#define TK_NK_BOOL                        123
-#define TK_NULL                           124
-#define TK_NK_VARIABLE                    125
-#define TK_NK_UNDERLINE                   126
-#define TK_ROWTS                          127
-#define TK_TBNAME                         128
-#define TK_QSTARTTS                       129
-#define TK_QENDTS                         130
-#define TK_WSTARTTS                       131
-#define TK_WENDTS                         132
-#define TK_WDURATION                      133
-#define TK_BETWEEN                        134
-#define TK_IS                             135
-#define TK_NK_LT                          136
-#define TK_NK_GT                          137
-#define TK_NK_LE                          138
-#define TK_NK_GE                          139
-#define TK_NK_NE                          140
-#define TK_MATCH                          141
-#define TK_NMATCH                         142
-#define TK_IN                             143
-#define TK_JOIN                           144
-#define TK_INNER                          145
-#define TK_SELECT                         146
-#define TK_DISTINCT                       147
-#define TK_WHERE                          148
-#define TK_PARTITION                      149
-#define TK_BY                             150
-#define TK_SESSION                        151
-#define TK_STATE_WINDOW                   152
-#define TK_SLIDING                        153
-#define TK_FILL                           154
-#define TK_VALUE                          155
-#define TK_NONE                           156
-#define TK_PREV                           157
-#define TK_LINEAR                         158
-#define TK_NEXT                           159
-#define TK_GROUP                          160
-#define TK_HAVING                         161
-#define TK_ORDER                          162
-#define TK_SLIMIT                         163
-#define TK_SOFFSET                        164
-#define TK_LIMIT                          165
-#define TK_OFFSET                         166
-#define TK_ASC                            167
-#define TK_DESC                           168
-#define TK_NULLS                          169
-#define TK_FIRST                          170
-#define TK_LAST                           171
+#define TK_ACCOUNTS                       116
+#define TK_APPS                           117
+#define TK_CONNECTIONS                    118
+#define TK_LICENCE                        119
+#define TK_QUERIES                        120
+#define TK_SCORES                         121
+#define TK_TOPICS                         122
+#define TK_VARIABLES                      123
+#define TK_LIKE                           124
+#define TK_INDEX                          125
+#define TK_FULLTEXT                       126
+#define TK_FUNCTION                       127
+#define TK_INTERVAL                       128
+#define TK_TOPIC                          129
+#define TK_AS                             130
+#define TK_DESC                           131
+#define TK_DESCRIBE                       132
+#define TK_RESET                          133
+#define TK_QUERY                          134
+#define TK_EXPLAIN                        135
+#define TK_ANALYZE                        136
+#define TK_VERBOSE                        137
+#define TK_NK_BOOL                        138
+#define TK_RATIO                          139
+#define TK_COMPACT                        140
+#define TK_VNODES                         141
+#define TK_IN                             142
+#define TK_OUTPUTTYPE                     143
+#define TK_AGGREGATE                      144
+#define TK_BUFSIZE                        145
+#define TK_STREAM                         146
+#define TK_INTO                           147
+#define TK_KILL                           148
+#define TK_CONNECTION                     149
+#define TK_MERGE                          150
+#define TK_VGROUP                         151
+#define TK_REDISTRIBUTE                   152
+#define TK_SPLIT                          153
+#define TK_SYNCDB                         154
+#define TK_NULL                           155
+#define TK_NK_VARIABLE                    156
+#define TK_NK_UNDERLINE                   157
+#define TK_ROWTS                          158
+#define TK_TBNAME                         159
+#define TK_QSTARTTS                       160
+#define TK_QENDTS                         161
+#define TK_WSTARTTS                       162
+#define TK_WENDTS                         163
+#define TK_WDURATION                      164
+#define TK_BETWEEN                        165
+#define TK_IS                             166
+#define TK_NK_LT                          167
+#define TK_NK_GT                          168
+#define TK_NK_LE                          169
+#define TK_NK_GE                          170
+#define TK_NK_NE                          171
+#define TK_MATCH                          172
+#define TK_NMATCH                         173
+#define TK_JOIN                           174
+#define TK_INNER                          175
+#define TK_SELECT                         176
+#define TK_DISTINCT                       177
+#define TK_WHERE                          178
+#define TK_PARTITION                      179
+#define TK_BY                             180
+#define TK_SESSION                        181
+#define TK_STATE_WINDOW                   182
+#define TK_SLIDING                        183
+#define TK_FILL                           184
+#define TK_VALUE                          185
+#define TK_NONE                           186
+#define TK_PREV                           187
+#define TK_LINEAR                         188
+#define TK_NEXT                           189
+#define TK_GROUP                          190
+#define TK_HAVING                         191
+#define TK_ORDER                          192
+#define TK_SLIMIT                         193
+#define TK_SOFFSET                        194
+#define TK_LIMIT                          195
+#define TK_OFFSET                         196
+#define TK_ASC                            197
+#define TK_NULLS                          198
+#define TK_FIRST                          199
+#define TK_LAST                           200
 
 #define TK_NK_SPACE                       300
 #define TK_NK_COMMENT                     301
@@ -200,10 +229,9 @@
 #define TK_NK_COLON                       500
 #define TK_NK_BITNOT                      501
 #define TK_INSERT                         502
-#define TK_INTO                           503
 #define TK_NOW                            504
 #define TK_VALUES                         507
-#define TK_IMPORT                         507
+#define TK_IMPORT                         509
 #define TK_NK_SEMI                        508
 
 #define TK_NK_NIL                         65535

include/dnode/mgmt/dnode.h

@@ -46,11 +46,12 @@ typedef struct {
   char      localFqdn[TSDB_FQDN_LEN];
   char      firstEp[TSDB_EP_LEN];
   char      secondEp[TSDB_EP_LEN];
-  SDiskCfg *pDisks;
+  SDiskCfg *disks;
   int32_t   numOfDisks;
+  int8_t    ntype;
 } SDnodeOpt;
 
-typedef enum { DND_EVENT_START, DND_EVENT_STOP = 1, DND_EVENT_RELOAD } EDndEvent;
+typedef enum { DND_EVENT_START, DND_EVENT_STOP = 1, DND_EVENT_CHILD } EDndEvent;
 
 /**
  * @brief Initialize and start the dnode.

include/libs/command/command.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cmdnodes.h"
+#include "tmsg.h"
+#include "plannodes.h"
+
+int32_t qExecCommand(SNode* pStmt, SRetrieveTableRsp** pRsp);
+
+int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp);
+
+

include/libs/executor/executor.h

@@ -37,6 +37,11 @@ typedef struct SReadHandle {
 #define STREAM_DATA_TYPE_SUBMIT_BLOCK 0x1
 #define STREAM_DATA_TYPE_SSDATA_BLOCK 0x2
 
+typedef enum {
+  OPTR_EXEC_MODEL_BATCH  = 0x1,
+  OPTR_EXEC_MODEL_STREAM = 0x2,
+} EOPTR_EXEC_MODEL;
+
 /**
  * Create the exec task for streaming mode
  * @param pMsg
@@ -84,7 +89,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA
  * @return
  */
 int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan,
-                        qTaskInfo_t* pTaskInfo, DataSinkHandle* handle);
+                        qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, EOPTR_EXEC_MODEL model);
 
 /**
  * The main task execution function, including query on both table and multiple tables,

include/libs/function/function.h

@@ -41,6 +41,7 @@ typedef void (*FExecFinalize)(struct SqlFunctionCtx *pCtx);
 typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
 typedef struct SScalarFuncExecFuncs {
+  FExecGetEnv getEnv;
   FScalarExecProcess process;
 } SScalarFuncExecFuncs;
 
@@ -241,7 +242,6 @@ typedef struct tExprNode {
   };
 } tExprNode;
 
-void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree);
 void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *));
 
 typedef struct SAggFunctionInfo {
@@ -267,28 +267,6 @@ struct SScalarParam {
   int32_t            numOfRows;
 };
 
-typedef struct SMultiFunctionsDesc {
-  bool stableQuery;
-  bool groupbyColumn;
-  bool agg;
-  bool arithmeticOnAgg;
-  bool projectionQuery;
-  bool hasFilter;
-  bool onlyTagQuery;
-  bool orderProjectQuery;
-  bool globalMerge;
-  bool multigroupResult;
-  bool blockDistribution;
-  bool stateWindow;
-  bool timewindow;
-  bool sessionWindow;
-  bool topbotQuery;
-  bool interpQuery;
-  bool distinct;
-  bool join;
-  bool continueQuery;
-} SMultiFunctionsDesc;
-
 int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, SResultDataInfo* pInfo, int16_t extLength,
                           bool isSuperTable);
 
@@ -296,8 +274,6 @@ bool qIsValidUdf(SArray* pUdfInfo, const char* name, int32_t len, int32_t* funct
 
 tExprNode* exprTreeFromBinary(const void* data, size_t size);
 
-void extractFunctionDesc(SArray* pFunctionIdList, SMultiFunctionsDesc* pDesc);
-
 tExprNode* exprdup(tExprNode* pTree);
 
 void resetResultRowEntryResult(SqlFunctionCtx* pCtx, int32_t num);

include/libs/nodes/cmdnodes.h

@@ -20,8 +20,14 @@
 extern "C" {
 #endif
 
+#include "query.h"
 #include "querynodes.h"
 
+#define DESCRIBE_RESULT_COLS 4
+#define DESCRIBE_RESULT_FIELD_LEN (TSDB_COL_NAME_LEN - 1 + VARSTR_HEADER_SIZE)
+#define DESCRIBE_RESULT_TYPE_LEN (20 + VARSTR_HEADER_SIZE)
+#define DESCRIBE_RESULT_NOTE_LEN (8 + VARSTR_HEADER_SIZE)
+
 typedef struct SDatabaseOptions {
   ENodeType type;
   int32_t numOfBlocks;
@@ -32,7 +38,9 @@ typedef struct SDatabaseOptions {
   int32_t fsyncPeriod;
   int32_t maxRowsPerBlock;
   int32_t minRowsPerBlock;
-  int32_t keep;
+  int32_t keep0;
+  int32_t keep1;
+  int32_t keep2;
   int32_t precision;
   int32_t quorum;
   int32_t replica;
@@ -70,7 +78,9 @@ typedef struct SAlterDatabaseStmt {
 
 typedef struct STableOptions {
   ENodeType type;
-  int32_t keep;
+  int32_t keep0;
+  int32_t keep1;
+  int32_t keep2;
   int32_t ttl;
   char comments[TSDB_STB_COMMENT_LEN];
   SNodeList* pSma;
@@ -187,6 +197,12 @@ typedef struct SShowStmt {
   SNode* pTbNamePattern; // SValueNode
 } SShowStmt;
 
+typedef struct SShowCreatStmt {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+  char tableName[TSDB_TABLE_NAME_LEN];
+} SShowCreatStmt;
+
 typedef enum EIndexType {
   INDEX_TYPE_SMA = 1,
   INDEX_TYPE_FULLTEXT
@@ -247,6 +263,13 @@ typedef struct SAlterLocalStmt {
   char value[TSDB_DNODE_VALUE_LEN];
 } SAlterLocalStmt;
 
+typedef struct SDescribeStmt {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+  char tableName[TSDB_TABLE_NAME_LEN];
+  STableMeta* pMeta;
+} SDescribeStmt;
+
 #ifdef __cplusplus
 }
 #endif

include/libs/nodes/nodes.h

@@ -70,6 +70,7 @@ typedef enum ENodeType {
   QUERY_NODE_DATABASE_OPTIONS,
   QUERY_NODE_TABLE_OPTIONS,
   QUERY_NODE_INDEX_OPTIONS,
+  QUERY_NODE_EXPLAIN_OPTIONS,
 
   // Statement nodes are used in parser and planner module.
   QUERY_NODE_SET_OPERATOR,
@@ -99,6 +100,18 @@ typedef enum ENodeType {
   QUERY_NODE_CREATE_TOPIC_STMT,
   QUERY_NODE_DROP_TOPIC_STMT,
   QUERY_NODE_ALTER_LOCAL_STMT,
+  QUERY_NODE_EXPLAIN_STMT,
+  QUERY_NODE_DESCRIBE_STMT,
+  QUERY_NODE_RESET_QUERY_CACHE_STMT,
+  QUERY_NODE_COMPACT_STMT,
+  QUERY_NODE_CREATE_FUNCTION_STMT,
+  QUERY_NODE_DROP_FUNCTION_STMT,
+  QUERY_NODE_CREATE_STREAM_STMT,
+  QUERY_NODE_DROP_STREAM_STMT,
+  QUERY_NODE_MERGE_VGROUP_STMT,
+  QUERY_NODE_REDISTRIBUTE_VGROUP_STMT,
+  QUERY_NODE_SPLIT_VGROUP_STMT,
+  QUERY_NODE_SYNCDB_STMT,
   QUERY_NODE_SHOW_DATABASES_STMT,
   QUERY_NODE_SHOW_TABLES_STMT,
   QUERY_NODE_SHOW_STABLES_STMT,
@@ -111,6 +124,18 @@ typedef enum ENodeType {
   QUERY_NODE_SHOW_FUNCTIONS_STMT,
   QUERY_NODE_SHOW_INDEXES_STMT,
   QUERY_NODE_SHOW_STREAMS_STMT,
+  QUERY_NODE_SHOW_APPS_STMT,
+  QUERY_NODE_SHOW_CONNECTIONS_STMT,
+  QUERY_NODE_SHOW_LICENCE_STMT,
+  QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
+  QUERY_NODE_SHOW_CREATE_TABLE_STMT,
+  QUERY_NODE_SHOW_CREATE_STABLE_STMT, 
+  QUERY_NODE_SHOW_QUERIES_STMT,
+  QUERY_NODE_SHOW_SCORES_STMT,
+  QUERY_NODE_SHOW_TOPICS_STMT,
+  QUERY_NODE_SHOW_VARIABLE_STMT,
+  QUERY_NODE_KILL_CONNECTION_STMT,
+  QUERY_NODE_KILL_QUERY_STMT,
 
   // logic plan node
   QUERY_NODE_LOGIC_PLAN_SCAN,
@@ -211,6 +236,8 @@ int32_t nodesStringToNode(const char* pStr, SNode** pNode);
 int32_t nodesListToString(const SNodeList* pList, bool format, char** pStr, int32_t* pLen);
 int32_t nodesStringToList(const char* pStr, SNodeList** pList);
 
+int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len);
+
 #ifdef __cplusplus
 }
 #endif

include/libs/nodes/plannodes.h

@@ -49,6 +49,7 @@ typedef struct SScanLogicNode {
   STimeWindow scanRange;
   SName tableName;
   bool showRewrite;
+  double ratio;
 } SScanLogicNode;
 
 typedef struct SJoinLogicNode {
@@ -197,7 +198,7 @@ typedef struct STableScanPhysiNode {
   SScanPhysiNode scan;
   uint8_t scanFlag;         // denotes reversed scan of data or not
   STimeWindow scanRange;
-  SNode* pScanConditions;
+  double ratio;
 } STableScanPhysiNode;
 
 typedef STableScanPhysiNode STableSeqScanPhysiNode;
@@ -252,6 +253,7 @@ typedef struct SIntervalPhysiNode {
   int64_t    sliding;
   int8_t     intervalUnit;
   int8_t     slidingUnit;
+  uint8_t    precision;
   SFillNode* pFill;
 } SIntervalPhysiNode;
 
@@ -297,18 +299,23 @@ typedef struct SSubplan {
   SDataSinkNode* pDataSink;    // data of the subplan flow into the datasink
 } SSubplan;
 
-typedef enum EQueryMode {
-  QUERY_MODE_NORMAL = 1,
-  QUERY_MODE_EXPLAIN,
-  QUERY_MODE_EXPLAIN_AN
-} EQueryMode;
+typedef enum EExplainMode {
+  EXPLAIN_MODE_DISABLE = 1,
+  EXPLAIN_MODE_STATIC,
+  EXPLAIN_MODE_ANALYZE
+} EExplainMode;
+
+typedef struct SExplainInfo {
+  EExplainMode mode;
+  bool verbose;
+} SExplainInfo;
 
 typedef struct SQueryPlan {
   ENodeType type;
   uint64_t queryId;
   int32_t numOfSubplans;
-
   SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0.
+  SExplainInfo explainInfo;
 } SQueryPlan;
 
 #ifdef __cplusplus

include/libs/nodes/querynodes.h

@@ -131,6 +131,7 @@ typedef struct SRealTableNode {
   struct STableMeta* pMeta;
   SVgroupsInfo* pVgroupList;
   char useDbName[TSDB_DB_NAME_LEN];
+  double ratio;
 } SRealTableNode;
 
 typedef struct STempTableNode {
@@ -282,6 +283,19 @@ typedef struct SVnodeModifOpStmt {
   const char* sql;                 // current sql statement position
 } SVnodeModifOpStmt;
 
+typedef struct SExplainOptions {
+  ENodeType type;
+  bool verbose;
+  double ratio;
+} SExplainOptions;
+
+typedef struct SExplainStmt {
+  ENodeType type;
+  bool analyze;
+  SExplainOptions* pOptions;
+  SNode* pQuery;
+} SExplainStmt;
+
 void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);
 void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext);
 
@@ -300,6 +314,7 @@ bool nodesIsTimeorderQuery(const SNode* pQuery);
 bool nodesIsTimelineQuery(const SNode* pQuery);
 
 void* nodesGetValueFromNode(SValueNode *pNode);
+char* nodesGetStrValueFromNode(SValueNode *pNode);
 
 #ifdef __cplusplus
 }

include/libs/parser/parser.h

@@ -55,6 +55,7 @@ typedef struct SQuery {
   SArray* pDbList;
   SArray* pTableList;
   bool showRewrite;
+  bool localCmd;
 } SQuery;
 
 int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery);

include/libs/qcom/query.h

@@ -53,6 +53,7 @@ typedef struct SIndexMeta {
 
 } SIndexMeta;
 
+
 /*
  * ASSERT(sizeof(SCTableMeta) == 24)
  * ASSERT(tableType == TSDB_CHILD_TABLE)
@@ -235,6 +236,11 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
     }                                                                    \
   } while (0)
 
+#define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
+#define QRY_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
+#define QRY_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
+
+
 #ifdef __cplusplus
 }
 #endif

include/libs/scalar/scalar.h

@@ -70,6 +70,13 @@ int32_t ltrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut
 int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
+bool getTimePseudoFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
+
+int32_t winStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t winEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t winDurFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t qStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t qEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
 #ifdef __cplusplus
 }

include/libs/wal/wal.h

@@ -61,16 +61,16 @@ extern "C" {
     }                                                             \
   }
 
-#define WAL_HEAD_VER     0
+#define WAL_HEAD_VER 0
 #define WAL_NOSUFFIX_LEN 20
-#define WAL_SUFFIX_AT    (WAL_NOSUFFIX_LEN + 1)
-#define WAL_LOG_SUFFIX   "log"
+#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1)
+#define WAL_LOG_SUFFIX "log"
 #define WAL_INDEX_SUFFIX "idx"
-#define WAL_REFRESH_MS   1000
-#define WAL_MAX_SIZE     (TSDB_MAX_WAL_SIZE + sizeof(SWalHead))
-#define WAL_PATH_LEN     (TSDB_FILENAME_LEN + 12)
-#define WAL_FILE_LEN     (WAL_PATH_LEN + 32)
-#define WAL_MAGIC        0xFAFBFCFDULL
+#define WAL_REFRESH_MS 1000
+#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead))
+#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12)
+#define WAL_FILE_LEN (WAL_PATH_LEN + 32)
+#define WAL_MAGIC 0xFAFBFCFDULL
 
 #define WAL_CUR_FAILED 1
 
@@ -150,14 +150,15 @@ typedef struct SWal {
 } SWal;  // WAL HANDLE
 
 typedef struct SWalReadHandle {
-  SWal     *pWal;
-  TdFilePtr pReadLogTFile;
-  TdFilePtr pReadIdxTFile;
-  int64_t   curFileFirstVer;
-  int64_t   curVersion;
-  int64_t   capacity;
-  int64_t   status;  // if cursor valid
-  SWalHead *pHead;
+  SWal         *pWal;
+  TdFilePtr     pReadLogTFile;
+  TdFilePtr     pReadIdxTFile;
+  int64_t       curFileFirstVer;
+  int64_t       curVersion;
+  int64_t       capacity;
+  int64_t       status;  // if cursor valid
+  TdThreadMutex mutex;
+  SWalHead     *pHead;
 } SWalReadHandle;
 #pragma pack(pop)
 
@@ -191,6 +192,7 @@ int32_t walEndSnapshot(SWal *);
 SWalReadHandle *walOpenReadHandle(SWal *);
 void            walCloseReadHandle(SWalReadHandle *);
 int32_t         walReadWithHandle(SWalReadHandle *pRead, int64_t ver);
+int32_t         walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead);
 
 // deprecated
 #if 0

include/os/os.h

@@ -42,6 +42,9 @@ extern "C" {
 #include <sys/types.h>
 #include <termios.h>
 #include <sys/statvfs.h>
+#include <sys/prctl.h>
+#include <sys/shm.h>
+#include <sys/wait.h>
 
 #if defined(DARWIN)
 #else
@@ -70,7 +73,6 @@ extern "C" {
 #include <wchar.h>
 #include <wctype.h>
 
-
 #include "osAtomic.h"
 #include "osDef.h"
 #include "osDir.h"
@@ -80,10 +82,12 @@ extern "C" {
 #include "osLz4.h"
 #include "osMath.h"
 #include "osMemory.h"
+#include "osProc.h"
 #include "osRand.h"
 #include "osThread.h"
 #include "osSemaphore.h"
 #include "osSignal.h"
+#include "osShm.h"
 #include "osSleep.h"
 #include "osSocket.h"
 #include "osString.h"

include/os/osMemory.h

@@ -32,13 +32,16 @@ extern "C" {
 void *taosMemoryMalloc(int32_t size);
 void *taosMemoryCalloc(int32_t num, int32_t size);
 void *taosMemoryRealloc(void *ptr, int32_t size);
+void *taosMemoryStrDup(void *ptr);
 void taosMemoryFree(const void *ptr);
 int32_t taosMemorySize(void *ptr);
 
 #define taosMemoryFreeClear(ptr) \
   do {                           \
-    taosMemoryFree(ptr);         \
-    (ptr)=NULL;                  \
+    if (ptr) {                   \
+      taosMemoryFree(ptr);       \
+      (ptr) = NULL;              \
+    }                            \
   } while (0)
 
 #ifdef __cplusplus

include/os/osProc.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_OS_PROC_H_
+#define _TD_OS_PROC_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t taosNewProc(char **args);
+void    taosWaitProc(int32_t pid);
+void    taosKillProc(int32_t pid);
+bool    taosProcExist(int32_t pid);
+void    taosSetProcName(int32_t argc, char **argv, const char *name);
+void    taosSetProcPath(int32_t argc, char **argv);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_OS_PROC_H_*/

include/os/osShm.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_OS_SHM_H_
+#define _TD_OS_SHM_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+  int32_t id;
+  int32_t size;
+  void*   ptr;
+} SShm;
+
+int32_t taosCreateShm(SShm *pShm, int32_t shmsize) ;
+void    taosDropShm(SShm *pShm);
+int32_t taosAttachShm(SShm *pShm);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_OS_SHM_H_*/

include/os/osSignal.h

@@ -49,6 +49,8 @@ void taosSetSignal(int32_t signum, FSignalHandler sigfp);
 void taosIgnSignal(int32_t signum);
 void taosDflSignal(int32_t signum);
 
+void taosKillChildOnParentStopped();
+
 #ifdef __cplusplus
 }
 #endif

include/os/osThread.h

@@ -32,6 +32,9 @@ typedef pthread_once_t TdThreadOnce;
 typedef pthread_rwlockattr_t TdThreadRwlockAttr;
 typedef pthread_cond_t TdThreadCond;
 typedef pthread_condattr_t TdThreadCondAttr;
+typedef pthread_key_t TdThreadKey;
+typedef pthread_barrier_t TdThreadBarrier;
+typedef pthread_barrierattr_t TdThreadBarrierAttr;
 
 #define taosThreadCleanupPush pthread_cleanup_push
 #define taosThreadCleanupPop pthread_cleanup_pop
@@ -49,68 +52,180 @@ typedef pthread_condattr_t TdThreadCondAttr;
     #define pthread_rwlockattr_t PTHREAD_RWLOCKATTR_T_TYPE_TAOS_FORBID
     #define pthread_cond_t PTHREAD_COND_T_TYPE_TAOS_FORBID
     #define pthread_condattr_t PTHREAD_CONDATTR_T_TYPE_TAOS_FORBID
-    #define pthread_spin_init PTHREAD_SPIN_INIT_FUNC_TAOS_FORBID
-    #define pthread_mutex_init PTHREAD_MUTEX_INIT_FUNC_TAOS_FORBID
-    #define pthread_spin_destroy PTHREAD_SPIN_DESTROY_FUNC_TAOS_FORBID
-    #define pthread_mutex_destroy PTHREAD_MUTEX_DESTROY_FUNC_TAOS_FORBID
-    #define pthread_spin_lock PTHREAD_SPIN_LOCK_FUNC_TAOS_FORBID
-    #define pthread_mutex_lock PTHREAD_MUTEX_LOCK_FUNC_TAOS_FORBID
-    #define pthread_spin_unlock PTHREAD_SPIN_UNLOCK_FUNC_TAOS_FORBID
-    #define pthread_mutex_unlock PTHREAD_MUTEX_UNLOCK_FUNC_TAOS_FORBID
-    #define pthread_rwlock_rdlock PTHREAD_RWLOCK_RDLOCK_FUNC_TAOS_FORBID
-    #define pthread_rwlock_wrlock PTHREAD_RWLOCK_WRLOCK_FUNC_TAOS_FORBID
-    #define pthread_rwlock_unlock PTHREAD_RWLOCK_UNLOCK_FUNC_TAOS_FORBID
-    #define pthread_testcancel PTHREAD_TESTCANCEL_FUNC_TAOS_FORBID
-    #define pthread_attr_init PTHREAD_ATTR_INIT_FUNC_TAOS_FORBID
+    #define pthread_key_t PTHREAD_KEY_T_TYPE_TAOS_FORBID
+    #define pthread_barrier_t PTHREAD_BARRIER_T_TYPE_TAOS_FORBID
+    #define pthread_barrierattr_t PTHREAD_BARRIERATTR_T_TYPE_TAOS_FORBID
     #define pthread_create PTHREAD_CREATE_FUNC_TAOS_FORBID
-    #define pthread_once PTHREAD_ONCE_FUNC_TAOS_FORBID
-    #define pthread_attr_setdetachstate PTHREAD_ATTR_SETDETACHSTATE_FUNC_TAOS_FORBID
     #define pthread_attr_destroy PTHREAD_ATTR_DESTROY_FUNC_TAOS_FORBID
-    #define pthread_join PTHREAD_JOIN_FUNC_TAOS_FORBID
-    #define pthread_rwlock_init PTHREAD_RWLOCK_INIT_FUNC_TAOS_FORBID
-    #define pthread_rwlock_destroy PTHREAD_RWLOCK_DESTROY_FUNC_TAOS_FORBID
-    #define pthread_cond_signal PTHREAD_COND_SIGNAL_FUNC_TAOS_FORBID
+    #define pthread_attr_getdetachstate PTHREAD_ATTR_GETDETACHSTATE_FUNC_TAOS_FORBID
+    #define pthread_attr_getinheritsched PTHREAD_ATTR_GETINHERITSCHED_FUNC_TAOS_FORBID
+    #define pthread_attr_getschedparam PTHREAD_ATTR_GETSCHEDPARAM_FUNC_TAOS_FORBID
+    #define pthread_attr_getschedpolicy PTHREAD_ATTR_GETSCHEDPOLICY_FUNC_TAOS_FORBID
+    #define pthread_attr_getscope PTHREAD_ATTR_GETSCOPE_FUNC_TAOS_FORBID
+    #define pthread_attr_getstacksize PTHREAD_ATTR_GETSTACKSIZE_FUNC_TAOS_FORBID
+    #define pthread_attr_init PTHREAD_ATTR_INIT_FUNC_TAOS_FORBID
+    #define pthread_attr_setdetachstate PTHREAD_ATTR_SETDETACHSTATE_FUNC_TAOS_FORBID
+    #define pthread_attr_setinheritsched PTHREAD_ATTR_SETINHERITSCHED_FUNC_TAOS_FORBID
+    #define pthread_attr_setschedparam PTHREAD_ATTR_SETSCHEDPARAM_FUNC_TAOS_FORBID
+    #define pthread_attr_setschedpolicy PTHREAD_ATTR_SETSCHEDPOLICY_FUNC_TAOS_FORBID
+    #define pthread_attr_setscope PTHREAD_ATTR_SETSCOPE_FUNC_TAOS_FORBID
+    #define pthread_attr_setstacksize PTHREAD_ATTR_SETSTACKSIZE_FUNC_TAOS_FORBID
+    #define pthread_barrier_destroy PTHREAD_BARRIER_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_barrier_init PTHREAD_BARRIER_INIT_FUNC_TAOS_FORBID
+    #define pthread_barrier_wait PTHREAD_BARRIER_WAIT_FUNC_TAOS_FORBID
+    #define pthread_barrierattr_destroy PTHREAD_BARRIERATTR_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_barrierattr_getpshared PTHREAD_BARRIERATTR_GETPSHARED_FUNC_TAOS_FORBID
+    #define pthread_barrierattr_init PTHREAD_BARRIERATTR_INIT_FUNC_TAOS_FORBID
+    #define pthread_barrierattr_setpshared PTHREAD_BARRIERATTR_SETPSHARED_FUNC_TAOS_FORBID
+    #define pthread_cancel PTHREAD_CANCEL_FUNC_TAOS_FORBID
+    #define pthread_cond_destroy PTHREAD_COND_DESTROY_FUNC_TAOS_FORBID
     #define pthread_cond_init PTHREAD_COND_INIT_FUNC_TAOS_FORBID
+    #define pthread_cond_signal PTHREAD_COND_SIGNAL_FUNC_TAOS_FORBID
     #define pthread_cond_broadcast PTHREAD_COND_BROADCAST_FUNC_TAOS_FORBID
-    #define pthread_cond_destroy PTHREAD_COND_DESTROY_FUNC_TAOS_FORBID
     #define pthread_cond_wait PTHREAD_COND_WAIT_FUNC_TAOS_FORBID
-    #define pthread_self PTHREAD_SELF_FUNC_TAOS_FORBID
+    #define pthread_cond_timedwait PTHREAD_COND_TIMEDWAIT_FUNC_TAOS_FORBID
+    #define pthread_condattr_destroy PTHREAD_CONDATTR_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_condattr_getpshared PTHREAD_CONDATTR_GETPSHARED_FUNC_TAOS_FORBID
+    #define pthread_condattr_init PTHREAD_CONDATTR_INIT_FUNC_TAOS_FORBID
+    #define pthread_condattr_setpshared PTHREAD_CONDATTR_SETPSHARED_FUNC_TAOS_FORBID
+    #define pthread_detach PTHREAD_DETACH_FUNC_TAOS_FORBID
     #define pthread_equal PTHREAD_EQUAL_FUNC_TAOS_FORBID
-    #define pthread_sigmask PTHREAD_SIGMASK_FUNC_TAOS_FORBID
-    #define pthread_cancel PTHREAD_CANCEL_FUNC_TAOS_FORBID
+    #define pthread_exit PTHREAD_EXIT_FUNC_TAOS_FORBID
+    #define pthread_getschedparam PTHREAD_GETSCHEDPARAM_FUNC_TAOS_FORBID
+    #define pthread_getspecific PTHREAD_GETSPECIFIC_FUNC_TAOS_FORBID
+    #define pthread_join PTHREAD_JOIN_FUNC_TAOS_FORBID
+    #define pthread_key_create PTHREAD_KEY_CREATE_FUNC_TAOS_FORBID
+    #define pthread_key_delete PTHREAD_KEY_DELETE_FUNC_TAOS_FORBID
     #define pthread_kill PTHREAD_KILL_FUNC_TAOS_FORBID
+    #define pthread_mutex_consistent PTHREAD_MUTEX_CONSISTENT_FUNC_TAOS_FORBID
+    #define pthread_mutex_destroy PTHREAD_MUTEX_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_mutex_init PTHREAD_MUTEX_INIT_FUNC_TAOS_FORBID
+    #define pthread_mutex_lock PTHREAD_MUTEX_LOCK_FUNC_TAOS_FORBID
+    #define pthread_mutex_timedlock PTHREAD_MUTEX_TIMEDLOCK_FUNC_TAOS_FORBID
+    #define pthread_mutex_trylock PTHREAD_MUTEX_TRYLOCK_FUNC_TAOS_FORBID
+    #define pthread_mutex_unlock PTHREAD_MUTEX_UNLOCK_FUNC_TAOS_FORBID
+    #define pthread_mutexattr_destroy PTHREAD_MUTEXATTR_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_mutexattr_getpshared PTHREAD_MUTEXATTR_GETPSHARED_FUNC_TAOS_FORBID
+    #define pthread_mutexattr_getrobust PTHREAD_MUTEXATTR_GETROBUST_FUNC_TAOS_FORBID
+    #define pthread_mutexattr_gettype PTHREAD_MUTEXATTR_GETTYPE_FUNC_TAOS_FORBID
+    #define pthread_mutexattr_init PTHREAD_MUTEXATTR_INIT_FUNC_TAOS_FORBID
+    #define pthread_mutexattr_setpshared PTHREAD_MUTEXATTR_SETPSHARED_FUNC_TAOS_FORBID
+    #define pthread_mutexattr_setrobust PTHREAD_MUTEXATTR_SETROBUST_FUNC_TAOS_FORBID
+    #define pthread_mutexattr_settype PTHREAD_MUTEXATTR_SETTYPE_FUNC_TAOS_FORBID
+    #define pthread_once PTHREAD_ONCE_FUNC_TAOS_FORBID
+    #define pthread_rwlock_destroy PTHREAD_RWLOCK_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_rwlock_init PTHREAD_RWLOCK_INIT_FUNC_TAOS_FORBID
+    #define pthread_rwlock_rdlock PTHREAD_RWLOCK_RDLOCK_FUNC_TAOS_FORBID
+    #define pthread_rwlock_timedrdlock PTHREAD_RWLOCK_TIMEDRDLOCK_FUNC_TAOS_FORBID
+    #define pthread_rwlock_timedwrlock PTHREAD_RWLOCK_TIMEDWRLOCK_FUNC_TAOS_FORBID
+    #define pthread_rwlock_tryrdlock PTHREAD_RWLOCK_TRYRDLOCK_FUNC_TAOS_FORBID
+    #define pthread_rwlock_trywrlock PTHREAD_RWLOCK_TRYWRLOCK_FUNC_TAOS_FORBID
+    #define pthread_rwlock_unlock PTHREAD_RWLOCK_UNLOCK_FUNC_TAOS_FORBID
+    #define pthread_rwlock_wrlock PTHREAD_RWLOCK_WRLOCK_FUNC_TAOS_FORBID
+    #define pthread_rwlockattr_destroy PTHREAD_RWLOCKATTR_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_rwlockattr_getpshared PTHREAD_RWLOCKATTR_GETPSHARED_FUNC_TAOS_FORBID
+    #define pthread_rwlockattr_init PTHREAD_RWLOCKATTR_INIT_FUNC_TAOS_FORBID
+    #define pthread_rwlockattr_setpshared PTHREAD_RWLOCKATTR_SETPSHARED_FUNC_TAOS_FORBID
+    #define pthread_self PTHREAD_SELF_FUNC_TAOS_FORBID
+    #define pthread_setcancelstate PTHREAD_SETCANCELSTATE_FUNC_TAOS_FORBID
+    #define pthread_setcanceltype PTHREAD_SETCANCELTYPE_FUNC_TAOS_FORBID
+    #define pthread_setschedparam PTHREAD_SETSCHEDPARAM_FUNC_TAOS_FORBID
+    #define pthread_setspecific PTHREAD_SETSPECIFIC_FUNC_TAOS_FORBID
+    #define pthread_spin_destroy PTHREAD_SPIN_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_spin_init PTHREAD_SPIN_INIT_FUNC_TAOS_FORBID
+    #define pthread_spin_lock PTHREAD_SPIN_LOCK_FUNC_TAOS_FORBID
+    #define pthread_spin_trylock PTHREAD_SPIN_TRYLOCK_FUNC_TAOS_FORBID
+    #define pthread_spin_unlock PTHREAD_SPIN_UNLOCK_FUNC_TAOS_FORBID
+    #define pthread_testcancel PTHREAD_TESTCANCEL_FUNC_TAOS_FORBID
+    #define pthread_sigmask PTHREAD_SIGMASK_FUNC_TAOS_FORBID
+    #define sigwait SIGWAIT_FUNC_TAOS_FORBID
 #endif
 
-int32_t taosThreadSpinInit(TdThreadSpinlock *lock, int pshared);
-int32_t taosThreadMutexInit(TdThreadMutex *mutex, const TdThreadMutexAttr *attr);
-int32_t taosThreadSpinDestroy(TdThreadSpinlock *lock);
+int32_t taosThreadCreate(TdThread * tid, const TdThreadAttr * attr, void *(*start)(void *), void *arg);
+int32_t taosThreadAttrDestroy(TdThreadAttr * attr);
+int32_t taosThreadAttrGetDetachState(const TdThreadAttr * attr, int32_t *detachstate);
+int32_t taosThreadAttrGetInheritSched(const TdThreadAttr * attr, int32_t *inheritsched);
+int32_t taosThreadAttrGetSchedParam(const TdThreadAttr * attr, struct sched_param *param);
+int32_t taosThreadAttrGetSchedPolicy(const TdThreadAttr * attr, int32_t *policy);
+int32_t taosThreadAttrGetScope(const TdThreadAttr * attr, int32_t *contentionscope);
+int32_t taosThreadAttrGetStackSize(const TdThreadAttr * attr, size_t * stacksize);
+int32_t taosThreadAttrInit(TdThreadAttr * attr);
+int32_t taosThreadAttrSetDetachState(TdThreadAttr * attr, int32_t detachstate);
+int32_t taosThreadAttrSetInheritSched(TdThreadAttr * attr, int32_t inheritsched);
+int32_t taosThreadAttrSetSchedParam(TdThreadAttr * attr, const struct sched_param *param);
+int32_t taosThreadAttrSetSchedPolicy(TdThreadAttr * attr, int32_t policy);
+int32_t taosThreadAttrSetScope(TdThreadAttr * attr, int32_t contentionscope);
+int32_t taosThreadAttrSetStackSize(TdThreadAttr * attr, size_t stacksize);
+int32_t taosThreadBarrierDestroy(TdThreadBarrier * barrier);
+int32_t taosThreadBarrierInit(TdThreadBarrier * barrier, const TdThreadBarrierAttr * attr, uint32_t count);
+int32_t taosThreadBarrierWait(TdThreadBarrier * barrier);
+int32_t taosThreadBarrierAttrDestroy(TdThreadBarrierAttr * attr);
+int32_t taosThreadBarrierAttrGetPshared(const TdThreadBarrierAttr * attr, int32_t *pshared);
+int32_t taosThreadBarrierAttrInit(TdThreadBarrierAttr * attr);
+int32_t taosThreadBarrierAttrSetPshared(TdThreadBarrierAttr * attr, int32_t pshared);
+int32_t taosThreadCancel(TdThread thread);
+int32_t taosThreadCondDestroy(TdThreadCond * cond);
+int32_t taosThreadCondInit(TdThreadCond * cond, const TdThreadCondAttr * attr);
+int32_t taosThreadCondSignal(TdThreadCond * cond);
+int32_t taosThreadCondBroadcast(TdThreadCond * cond);
+int32_t taosThreadCondWait(TdThreadCond * cond, TdThreadMutex * mutex);
+int32_t taosThreadCondTimedWait(TdThreadCond * cond, TdThreadMutex * mutex, const struct timespec *abstime);
+int32_t taosThreadCondAttrDestroy(TdThreadCondAttr * attr);
+int32_t taosThreadCondAttrGetPshared(const TdThreadCondAttr * attr, int32_t *pshared);
+int32_t taosThreadCondAttrInit(TdThreadCondAttr * attr);
+int32_t taosThreadCondAttrSetPshared(TdThreadCondAttr * attr, int32_t pshared);
+int32_t taosThreadDetach(TdThread thread);
+int32_t taosThreadEqual(TdThread t1, TdThread t2);
+void taosThreadExit(void *valuePtr);
+int32_t taosThreadGetSchedParam(TdThread thread, int32_t *policy, struct sched_param *param);
+void *taosThreadGetSpecific(TdThreadKey key);
+int32_t taosThreadJoin(TdThread thread, void **valuePtr);
+int32_t taosThreadKeyCreate(TdThreadKey * key, void(*destructor)(void *));
+int32_t taosThreadKeyDelete(TdThreadKey key);
+int32_t taosThreadKill(TdThread thread, int32_t sig);
+int32_t taosThreadMutexConsistent(TdThreadMutex* mutex);
 int32_t taosThreadMutexDestroy(TdThreadMutex * mutex);
-int32_t taosThreadSpinLock(TdThreadSpinlock *lock);
-int32_t taosThreadMutexLock(TdThreadMutex *mutex);
-int32_t taosThreadRwlockRdlock(TdThreadRwlock *rwlock);
-int32_t taosThreadSpinUnlock(TdThreadSpinlock *lock);
-int32_t taosThreadMutexUnlock(TdThreadMutex *mutex);
-int32_t taosThreadRwlockWrlock(TdThreadRwlock *rwlock);
-int32_t taosThreadRwlockUnlock(TdThreadRwlock *rwlock);
-void taosThreadTestCancel(void);
-int32_t taosThreadAttrInit(TdThreadAttr *attr);
-int32_t taosThreadCreate(TdThread *tid, const TdThreadAttr *attr, void*(*start)(void*), void *arg);
-int32_t taosThreadOnce(TdThreadOnce *onceControl, void(*initRoutine)(void));
-int32_t taosThreadAttrSetDetachState(TdThreadAttr *attr, int32_t detachState);
-int32_t taosThreadAttrDestroy(TdThreadAttr *attr);
-int32_t taosThreadJoin(TdThread thread, void **pValue);
-int32_t taosThreadRwlockInit(TdThreadRwlock *rwlock, const TdThreadRwlockAttr *attr);
-int32_t taosThreadRwlockDestroy(TdThreadRwlock *rwlock);
-int32_t taosThreadCondSignal(TdThreadCond *cond);
-int32_t taosThreadCondInit(TdThreadCond *cond, const TdThreadCondAttr *attr);
-int32_t taosThreadCondBroadcast(TdThreadCond *cond);
-int32_t taosThreadCondDestroy(TdThreadCond *cond);
-int32_t taosThreadCondWait(TdThreadCond *cond, TdThreadMutex *mutex);
+int32_t taosThreadMutexInit(TdThreadMutex * mutex, const TdThreadMutexAttr * attr);
+int32_t taosThreadMutexLock(TdThreadMutex * mutex);
+int32_t taosThreadMutexTimedLock(TdThreadMutex * mutex, const struct timespec *abstime);
+int32_t taosThreadMutexTryLock(TdThreadMutex * mutex);
+int32_t taosThreadMutexUnlock(TdThreadMutex * mutex);
+int32_t taosThreadMutexAttrDestroy(TdThreadMutexAttr * attr);
+int32_t taosThreadMutexAttrGetPshared(const TdThreadMutexAttr * attr, int32_t *pshared);
+int32_t taosThreadMutexAttrGetRobust(const TdThreadMutexAttr * attr, int32_t * robust);
+int32_t taosThreadMutexAttrGetType(const TdThreadMutexAttr * attr, int32_t *kind);
+int32_t taosThreadMutexAttrInit(TdThreadMutexAttr * attr);
+int32_t taosThreadMutexAttrSetPshared(TdThreadMutexAttr * attr, int32_t pshared);
+int32_t taosThreadMutexAttrSetRobust(TdThreadMutexAttr * attr, int32_t robust);
+int32_t taosThreadMutexAttrSetType(TdThreadMutexAttr * attr, int32_t kind);
+int32_t taosThreadOnce(TdThreadOnce * onceControl, void(*initRoutine)(void));
+int32_t taosThreadRwlockDestroy(TdThreadRwlock * rwlock);
+int32_t taosThreadRwlockInit(TdThreadRwlock * rwlock, const TdThreadRwlockAttr * attr);
+int32_t taosThreadRwlockRdlock(TdThreadRwlock * rwlock);
+int32_t taosThreadRwlockTimedRdlock(TdThreadRwlock * rwlock, const struct timespec *abstime);
+int32_t taosThreadRwlockTimedWrlock(TdThreadRwlock * rwlock, const struct timespec *abstime);
+int32_t taosThreadRwlockTryRdlock(TdThreadRwlock * rwlock);
+int32_t taosThreadRwlockTryWrlock(TdThreadRwlock * rwlock);
+int32_t taosThreadRwlockUnlock(TdThreadRwlock * rwlock);
+int32_t taosThreadRwlockWrlock(TdThreadRwlock * rwlock);
+int32_t taosThreadRwlockAttrDestroy(TdThreadRwlockAttr * attr);
+int32_t taosThreadRwlockAttrGetPshared(const TdThreadRwlockAttr * attr, int32_t *pshared);
+int32_t taosThreadRwlockAttrInit(TdThreadRwlockAttr * attr);
+int32_t taosThreadRwlockAttrSetPshared(TdThreadRwlockAttr * attr, int32_t pshared);
 TdThread taosThreadSelf(void);
-int32_t taosThreadEqual(TdThread t1, TdThread t2);
-int32_t taosThreadSigmask(int how, sigset_t const *set, sigset_t *oset);
-int32_t taosThreadCancel(TdThread thread);
-int32_t taosThreadKill(TdThread thread, int sig);
+int32_t taosThreadSetCancelState(int32_t state, int32_t *oldstate);
+int32_t taosThreadSetCancelType(int32_t type, int32_t *oldtype);
+int32_t taosThreadSetSchedParam(TdThread thread, int32_t policy, const struct sched_param *param);
+int32_t taosThreadSetSpecific(TdThreadKey key, const void *value);
+int32_t taosThreadSpinDestroy(TdThreadSpinlock * lock);
+int32_t taosThreadSpinInit(TdThreadSpinlock * lock, int32_t pshared);
+int32_t taosThreadSpinLock(TdThreadSpinlock * lock);
+int32_t taosThreadSpinTrylock(TdThreadSpinlock * lock);
+int32_t taosThreadSpinUnlock(TdThreadSpinlock * lock);
+void taosThreadTestCancel(void);
+int32_t taosThreadSigMask(int32_t how, sigset_t const *set, sigset_t * oset);
+int32_t taosThreadSigWait(const sigset_t * set, int32_t *sig);
+
 #ifdef __cplusplus
 }
 #endif

include/os/osTime.h

@@ -20,6 +20,8 @@
 extern "C" {
 #endif
 
+#include <time.h>
+
 // If the error is in a third-party library, place this header file under the third-party library header file.
 // When you want to use this feature, you should find or add the same function in the following section.
 #ifndef ALLOW_FORBID_FUNC
@@ -44,6 +46,14 @@ extern "C" {
 #define MILLISECOND_PER_DAY    (MILLISECOND_PER_HOUR * 24)
 #define MILLISECOND_PER_WEEK   (MILLISECOND_PER_DAY * 7)
 
+#define NANOSECOND_PER_USEC   (1000L)
+#define NANOSECOND_PER_MSEC   (1000000L)
+#define NANOSECOND_PER_SEC    (1000000000L)
+#define NANOSECOND_PER_MINUTE (NANOSECOND_PER_SEC * 60)
+#define NANOSECOND_PER_HOUR   (NANOSECOND_PER_MINUTE * 60)
+#define NANOSECOND_PER_DAY    (NANOSECOND_PER_HOUR * 24)
+#define NANOSECOND_PER_WEEK   (NANOSECOND_PER_DAY * 7)
+
 int32_t taosGetTimeOfDay(struct timeval *tv);
 
 //@return timestamp in second

include/util/taoserror.h

@@ -16,7 +16,7 @@
 #ifndef _TD_UTIL_TAOS_ERROR_H_
 #define _TD_UTIL_TAOS_ERROR_H_
 
-#include "os.h"
+#include <stdint.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -78,6 +78,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_CFG_NOT_FOUND                 TAOS_DEF_ERROR_CODE(0, 0x010C)
 #define TSDB_CODE_INVALID_CFG                   TAOS_DEF_ERROR_CODE(0, 0x010D)
 #define TSDB_CODE_OUT_OF_SHM_MEM                TAOS_DEF_ERROR_CODE(0, 0x010E)
+#define TSDB_CODE_INVALID_SHM_ID                TAOS_DEF_ERROR_CODE(0, 0x010F)
 #define TSDB_CODE_REF_NO_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0110)
 #define TSDB_CODE_REF_FULL                      TAOS_DEF_ERROR_CODE(0, 0x0111)
 #define TSDB_CODE_REF_ID_REMOVED                TAOS_DEF_ERROR_CODE(0, 0x0112)
@@ -478,6 +479,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INVALID_ENDPOINT          TAOS_DEF_ERROR_CODE(0, 0x2613)
 #define TSDB_CODE_PAR_EXPRIE_STATEMENT          TAOS_DEF_ERROR_CODE(0, 0x2614)
 #define TSDB_CODE_PAR_INTERVAL_VALUE_TOO_SMALL  TAOS_DEF_ERROR_CODE(0, 0x2615)
+#define TSDB_CODE_PAR_DB_NOT_SPECIFIED          TAOS_DEF_ERROR_CODE(0, 0x2616)
+#define TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME   TAOS_DEF_ERROR_CODE(0, 0x2617)
+#define TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR  TAOS_DEF_ERROR_CODE(0, 0x2618)
 
 #ifdef __cplusplus
 }

include/util/tcoding.h

@@ -57,6 +57,8 @@ static FORCE_INLINE void *taosDecodeFixedI8(const void *buf, int8_t *value) {
   return POINTER_SHIFT(buf, sizeof(*value));
 }
 
+static FORCE_INLINE void *taosSkipFixedLen(const void *buf, size_t len) { return POINTER_SHIFT(buf, len); }
+
 // ---- Fixed U16
 static FORCE_INLINE int32_t taosEncodeFixedU16(void **buf, uint16_t value) {
   if (buf != NULL) {

include/util/tdef.h

@@ -224,6 +224,7 @@ typedef enum ELogicConditionType {
 
 #define TSDB_APP_NAME_LEN    TSDB_UNI_LEN
 #define TSDB_STB_COMMENT_LEN 1024
+
 /**
  *  In some scenarios uint16_t (0~65535) is used to store the row len.
  *  - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header.
@@ -303,13 +304,13 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_TOTAL_BLOCKS     10000
 #define TSDB_DEFAULT_TOTAL_BLOCKS 6
 
-#define TSDB_MIN_DAYS_PER_FILE     1
-#define TSDB_MAX_DAYS_PER_FILE     3650
-#define TSDB_DEFAULT_DAYS_PER_FILE 10
+#define TSDB_MIN_DAYS_PER_FILE     (1 * 1440)    // unit minute
+#define TSDB_MAX_DAYS_PER_FILE     (3650 * 1440)
+#define TSDB_DEFAULT_DAYS_PER_FILE (10 * 1440)
 
-#define TSDB_MIN_KEEP     1       // data in db to be reserved.
-#define TSDB_MAX_KEEP     365000  // data in db to be reserved.
-#define TSDB_DEFAULT_KEEP 3650    // ten years
+#define TSDB_MIN_KEEP     (1 * 1440)       // data in db to be reserved. unit minute
+#define TSDB_MAX_KEEP     (365000 * 1440) // data in db to be reserved.
+#define TSDB_DEFAULT_KEEP (3650 * 1440)    // ten years
 
 #define TSDB_MIN_MIN_ROW_FBLOCK     10
 #define TSDB_MAX_MIN_ROW_FBLOCK     1000
@@ -327,7 +328,7 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_FSYNC_PERIOD     180000  // millisecond
 #define TSDB_DEFAULT_FSYNC_PERIOD 3000    // three second
 
-#define TSDB_MIN_WAL_LEVEL     0
+#define TSDB_MIN_WAL_LEVEL     1
 #define TSDB_MAX_WAL_LEVEL     2
 #define TSDB_DEFAULT_WAL_LEVEL 1
 
@@ -381,6 +382,15 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_DB_DELAY     10
 #define TSDB_DEFAULT_DB_DELAY 2
 
+#define TSDB_DEFAULT_EXPLAIN_VERBOSE false
+
+#define TSDB_MIN_EXPLAIN_RATIO       0
+#define TSDB_MAX_EXPLAIN_RATIO       1
+#define TSDB_DEFAULT_EXPLAIN_RATIO   0.001
+
+#define TSDB_EXPLAIN_RESULT_ROW_SIZE 1024
+#define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY PLAN"
+
 #define TSDB_MAX_JOIN_TABLE_NUM 10
 #define TSDB_MAX_UNION_CLAUSE   5
 
@@ -471,6 +481,9 @@ enum {
 #define QND_VGID 1
 #define VND_VGID 0
 
+#define MAX_NUM_STR_SIZE 40
+
+
 #ifdef __cplusplus
 }
 #endif

include/util/tprocess.h

@@ -32,29 +32,25 @@ typedef void *(*ProcConsumeFp)(void *pParent, void *pHead, int16_t headLen, void
                                ProcFuncType ftype);
 
 typedef struct {
-  int32_t       childQueueSize;
   ProcConsumeFp childConsumeFp;
   ProcMallocFp  childMallocHeadFp;
   ProcFreeFp    childFreeHeadFp;
   ProcMallocFp  childMallocBodyFp;
   ProcFreeFp    childFreeBodyFp;
-  int32_t       parentQueueSize;
   ProcConsumeFp parentConsumeFp;
-  ProcMallocFp  parentdMallocHeadFp;
+  ProcMallocFp  parentMallocHeadFp;
   ProcFreeFp    parentFreeHeadFp;
   ProcMallocFp  parentMallocBodyFp;
   ProcFreeFp    parentFreeBodyFp;
-  bool          testFlag;
+  SShm          shm;
   void         *pParent;
   const char   *name;
+  bool          isChild;
 } SProcCfg;
 
 SProcObj *taosProcInit(const SProcCfg *pCfg);
 void      taosProcCleanup(SProcObj *pProc);
 int32_t   taosProcRun(SProcObj *pProc);
-void      taosProcStop(SProcObj *pProc);
-bool      taosProcIsChild(SProcObj *pProc);
-int32_t   taosProcChildId(SProcObj *pProc);
 int32_t   taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
                               ProcFuncType ftype);
 int32_t   taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,

source/client/CMakeLists.txt

@@ -8,7 +8,7 @@ target_include_directories(
 target_link_libraries(
     taos
     INTERFACE api
-    PRIVATE os util common transport nodes parser planner catalog scheduler function qcom
+    PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom
 )
 
 if(${BUILD_TEST})

source/client/inc/clientInt.h

@@ -136,7 +136,7 @@ typedef struct STscObj {
   uint32_t        connId;
   int32_t         connType;
   uint64_t        id;         // ref ID returned by taosAddRef
-  TdThreadMutex mutex;      // used to protect the operation on db
+  TdThreadMutex   mutex;      // used to protect the operation on db
   int32_t         numOfReqs;  // number of sqlObj bound to this connection
   SAppInstInfo*   pAppInfo;
 } STscObj;
@@ -152,7 +152,8 @@ typedef struct SResultColumn {
 typedef struct SReqResultInfo {
   const char*    pRspMsg;
   const char*    pData;
-  TAOS_FIELD*    fields;
+  TAOS_FIELD*    fields;     // todo, column names are not needed.
+  TAOS_FIELD*    userFields; // the fields info that return to user
   uint32_t       numOfCols;
   int32_t*       length;
   char**         convertBuf;
@@ -221,6 +222,7 @@ void  destroyRequest(SRequestObj* pRequest);
 
 char* getDbOfConnection(STscObj* pObj);
 void  setConnectionDB(STscObj* pTscObj, const char* db);
+void  resetConnectDB(STscObj* pTscObj);
 
 void taos_init_imp(void);
 int  taos_options_imp(TSDB_OPTION option, const char* str);

source/client/src/clientEnv.c

@@ -169,6 +169,7 @@ static void doFreeReqResultInfo(SReqResultInfo *pResInfo) {
   taosMemoryFreeClear(pResInfo->row);
   taosMemoryFreeClear(pResInfo->pCol);
   taosMemoryFreeClear(pResInfo->fields);
+  taosMemoryFreeClear(pResInfo->userFields);
 
   if (pResInfo->convertBuf != NULL) {
     for (int32_t i = 0; i < pResInfo->numOfCols; ++i) {
@@ -200,6 +201,9 @@ static void doDestroyRequest(void *p) {
     taosArrayDestroy(pRequest->body.showInfo.pArray);
   }
 
+  taosArrayDestroy(pRequest->tableList);
+  taosArrayDestroy(pRequest->dbList);
+
   deregisterRequest(pRequest);
   taosMemoryFreeClear(pRequest);
 }

source/client/src/clientImpl.c

 
 #include "clientInt.h"
 #include "clientLog.h"
+#include "command.h"
 #include "scheduler.h"
 #include "tdatablock.h"
 #include "tdef.h"
@@ -55,7 +56,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass,
   }
 
   char localDb[TSDB_DB_NAME_LEN] = {0};
-  if (db != NULL) {
+  if (db != NULL && strlen(db) > 0) {
     if (!validateDbName(db)) {
       terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH;
       return NULL;
@@ -163,6 +164,7 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery) {
     if ((*pQuery)->haveResultSet) {
       setResSchemaInfo(&pRequest->body.resInfo, (*pQuery)->pResSchema, (*pQuery)->numOfResCols);
     }
+
     TSWAP(pRequest->dbList, (*pQuery)->pDbList, SArray*);
     TSWAP(pRequest->tableList, (*pQuery)->pTableList, SArray*);
   }
@@ -170,7 +172,21 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery) {
   return code;
 }
 
+int32_t execLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
+  SRetrieveTableRsp* pRsp = NULL;
+  int32_t code = qExecCommand(pQuery->pRoot, &pRsp);
+  if (TSDB_CODE_SUCCESS == code && NULL != pRsp) {
+    code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp);
+  }
+  return code;
+}
+
 int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
+  // drop table if exists not_exists_table
+  if (NULL == pQuery->pCmdMsg) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   SCmdMsgInfo* pMsgInfo = pQuery->pCmdMsg;
   pRequest->type = pMsgInfo->msgType;
   pRequest->body.requestMsg = (SDataBuf){.pData = pMsgInfo->pMsg, .len = pMsgInfo->msgLen, .handle = NULL};
@@ -213,12 +229,24 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t
   assert(pSchema != NULL && numOfCols > 0);
 
   pResInfo->numOfCols = numOfCols;
-  pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(pSchema[0]));
+  pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD));
+  pResInfo->userFields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD));
 
   for (int32_t i = 0; i < pResInfo->numOfCols; ++i) {
     pResInfo->fields[i].bytes = pSchema[i].bytes;
-    pResInfo->fields[i].type = pSchema[i].type;
+    pResInfo->fields[i].type  = pSchema[i].type;
+
+    pResInfo->userFields[i].bytes = pSchema[i].bytes;
+    pResInfo->userFields[i].type  = pSchema[i].type;
+
+    if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR) {
+      pResInfo->userFields[i].bytes -= VARSTR_HEADER_SIZE;
+    } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
+      pResInfo->userFields[i].bytes = (pResInfo->userFields[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
+    }
+
     tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name));
+    tstrncpy(pResInfo->userFields[i].name, pSchema[i].name, tListLen(pResInfo->userFields[i].name));
   }
 }
 
@@ -259,7 +287,9 @@ SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) {
   CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
   CHECK_CODE_GOTO(parseSql(pRequest, false, &pQuery), _return);
 
-  if (pQuery->directRpc) {
+  if (pQuery->localCmd) {
+    CHECK_CODE_GOTO(execLocalCmd(pRequest, pQuery), _return);
+  } else if (pQuery->directRpc) {
     CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return);
   } else {
     CHECK_CODE_GOTO(getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList), _return);
@@ -464,9 +494,11 @@ static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) {
   taosMemoryFreeClear(pMsgBody->msgInfo.pData);
   taosMemoryFreeClear(pMsgBody);
 }
+
 bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType) {
   return msgType == TDMT_VND_QUERY_RSP || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP || msgType == TDMT_VND_QUERY_HEARTBEAT_RSP;
 }
+
 void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
   SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->ahandle;
   assert(pMsg->ahandle != NULL);
@@ -647,6 +679,11 @@ void* doFetchRow(SRequestObj* pRequest, bool setupOneRowPtr) {
       }
     }
 
+    if (pResultInfo->completed) {
+      pResultInfo->numOfRows = 0;
+      return NULL;
+    }
+
     SMsgSendInfo* body = buildMsgInfoImpl(pRequest);
 
     int64_t  transporterId = 0;
@@ -767,6 +804,16 @@ void setConnectionDB(STscObj* pTscObj, const char* db) {
   taosThreadMutexUnlock(&pTscObj->mutex);
 }
 
+void resetConnectDB(STscObj* pTscObj) {
+  if (pTscObj == NULL) {
+    return;
+  }
+
+  taosThreadMutexLock(&pTscObj->mutex);
+  pTscObj->db[0] = 0;
+  taosThreadMutexUnlock(&pTscObj->mutex);
+}
+
 int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) {
   assert(pResultInfo != NULL && pRsp != NULL);
 

source/client/src/clientMain.c

@@ -71,6 +71,12 @@ void taos_cleanup(void) {
   tscInfo("all local resources released");
 }
 
+setConfRet   taos_set_config(const char *config) {
+  // TODO
+  setConfRet ret = {SET_CONF_RET_SUCC, {0}};
+  return ret;
+}
+
 TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) {
   tscDebug("try to connect to %s:%u, user:%s db:%s", ip, port, user, db);
   if (user == NULL) {
@@ -140,7 +146,7 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
   }
 
   SReqResultInfo *pResInfo = &(((SRequestObj *)res)->body.resInfo);
-  return pResInfo->fields;
+  return pResInfo->userFields;
 }
 
 TAOS_RES *taos_query(TAOS *taos, const char *sql) {
@@ -257,6 +263,11 @@ int *taos_fetch_lengths(TAOS_RES *res) {
   return ((SRequestObj *)res)->body.resInfo.length;
 }
 
+TAOS_ROW *taos_result_block(TAOS_RES *res) {
+  // TODO
+  return NULL;
+}
+
 // todo intergrate with tDataTypes
 const char *taos_data_type(int type) {
   switch (type) {
@@ -353,6 +364,10 @@ bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) {
   return colDataIsNull_f(pCol->nullbitmap, row);
 }
 
+bool taos_is_update_query(TAOS_RES *res) {
+  return taos_num_fields(res) == 0;
+}
+
 int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
   if (res == NULL) {
     return 0;
@@ -376,6 +391,14 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
 
 int taos_validate_sql(TAOS *taos, const char *sql) { return true; }
 
+void taos_reset_current_db(TAOS *taos) {
+  if (taos == NULL) {
+    return;
+  }
+
+  resetConnectDB(taos);
+}
+
 const char *taos_get_server_info(TAOS *taos) {
   if (taos == NULL) {
     return NULL;
@@ -407,6 +430,11 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
     // TODO
 }
 
+int taos_load_table_info(TAOS *taos, const char *tableNameList) {
+  // TODO
+  return -1;
+}
+
 TAOS_STMT *taos_stmt_init(TAOS *taos) {
   // TODO
   return NULL;

source/client/src/tmq.c

@@ -78,6 +78,7 @@ struct tmq_t {
   STscObj*       pTscObj;
   tmq_commit_cb* commit_cb;
   int32_t        nextTopicIdx;
+  int8_t         epStatus;
   int32_t        waitingRequest;
   int32_t        readyRequest;
   SArray*        clientTopics;  // SArray<SMqClientTopic>
@@ -241,6 +242,10 @@ void tmq_list_destroy(tmq_list_t* list) {
   taosArrayDestroyEx(container, (void (*)(void*))taosMemoryFree);
 }
 
+static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) {
+  return sprintf(dst, "%s:%d", topicName, vg);
+}
+
 void tmqClearUnhandleMsg(tmq_t* tmq) {
   tmq_message_t* msg = NULL;
   while (1) {
@@ -251,6 +256,7 @@ void tmqClearUnhandleMsg(tmq_t* tmq) {
       break;
   }
 
+  msg = NULL;
   taosReadAllQitems(tmq->mqueue, tmq->qall);
   while (1) {
     taosGetQitem(tmq->qall, (void**)&msg);
@@ -306,6 +312,7 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
   pTmq->epoch = 0;
   pTmq->waitingRequest = 0;
   pTmq->readyRequest = 0;
+  pTmq->epStatus = 0;
   // set conf
   strcpy(pTmq->clientId, conf->clientId);
   strcpy(pTmq->groupId, conf->groupId);
@@ -783,7 +790,7 @@ void tmqShowMsg(tmq_message_t* tmq_message) {
   static bool noPrintSchema;
   char        pBuf[128];
   SMqPollRsp* pRsp = &tmq_message->msg;
-  int32_t     colNum = pRsp->schema->nCols;
+  int32_t     colNum = 2;
   if (!noPrintSchema) {
     printf("|");
     for (int32_t i = 0; i < colNum; i++) {
@@ -827,20 +834,21 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   SMqClientVg*    pVg = pParam->pVg;
   tmq_t*          tmq = pParam->tmq;
   if (code != 0) {
-    printf("msg discard, code:%x\n", code);
-    goto WRITE_QUEUE_FAIL;
+    tscWarn("msg discard, code:%x", code);
+    goto CREATE_MSG_FAIL;
   }
 
   int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch;
   int32_t tmqEpoch = atomic_load_32(&tmq->epoch);
   if (msgEpoch < tmqEpoch) {
+    /*printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);*/
     tsem_post(&tmq->rspSem);
-    printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);
+    tscWarn("discard rsp epoch %d, current epoch %d", msgEpoch, tmqEpoch);
     return 0;
   }
 
   if (msgEpoch != tmqEpoch) {
-    printf("mismatch rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);
+    tscWarn("mismatch rsp epoch %d, current epoch %d", msgEpoch, tmqEpoch);
   } else {
     atomic_sub_fetch_32(&tmq->waitingRequest, 1);
   }
@@ -867,7 +875,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   /*SMqConsumeRsp* pRsp = taosMemoryCalloc(1, sizeof(SMqConsumeRsp));*/
   tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t));
   if (pRsp == NULL) {
-    goto WRITE_QUEUE_FAIL;
+    goto CREATE_MSG_FAIL;
   }
   memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));
   tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->msg);
@@ -876,11 +884,16 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   // TODO: alloc mem
   /*pRsp->*/
   /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/
+#if 0
   if (pRsp->msg.numOfTopics == 0) {
     /*printf("no data\n");*/
     taosFreeQitem(pRsp);
-    goto WRITE_QUEUE_FAIL;
+    goto CREATE_MSG_FAIL;
   }
+#endif
+
+  tscError("tmq recv poll: vg %d, req offset %ld, rsp offset %ld", pParam->pVg->vgId, pRsp->msg.reqOffset,
+           pRsp->msg.rspOffset);
 
   pRsp->vg = pParam->pVg;
   taosWriteQitem(tmq->mqueue, pRsp);
@@ -888,7 +901,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   tsem_post(&tmq->rspSem);
   return 0;
 
-WRITE_QUEUE_FAIL:
+CREATE_MSG_FAIL:
   if (pParam->epoch == tmq->epoch) {
     atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
   }
@@ -898,20 +911,60 @@ WRITE_QUEUE_FAIL:
 
 bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
   /*printf("call update ep %d\n", epoch);*/
+  tscDebug("tmq update ep epoch %d to epoch %d", tmq->epoch, epoch);
   bool    set = false;
-  int32_t sz = taosArrayGetSize(pRsp->topics);
-  SArray* newTopics = taosArrayInit(sz, sizeof(SMqClientTopic));
-  for (int32_t i = 0; i < sz; i++) {
+  int32_t topicNumGet = taosArrayGetSize(pRsp->topics);
+  char    vgKey[TSDB_TOPIC_FNAME_LEN + 22];
+  SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic));
+  if (newTopics == NULL) {
+    return false;
+  }
+  SHashObj* pHash = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK);
+  if (pHash == NULL) {
+    taosArrayDestroy(newTopics);
+    return false;
+  }
+
+  // find topic, build hash
+  for (int32_t i = 0; i < topicNumGet; i++) {
     SMqClientTopic topic = {0};
     SMqSubTopicEp* pTopicEp = taosArrayGet(pRsp->topics, i);
+    taosHashClear(pHash);
     topic.topicName = strdup(pTopicEp->topic);
-    int32_t vgSz = taosArrayGetSize(pTopicEp->vgs);
-    topic.vgs = taosArrayInit(vgSz, sizeof(SMqClientVg));
-    for (int32_t j = 0; j < vgSz; j++) {
+
+    int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics);
+    for (int32_t j = 0; j < topicNumCur; j++) {
+      // find old topic
+      SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, j);
+      if (pTopicCur->vgs && strcmp(pTopicCur->topicName, pTopicEp->topic) == 0) {
+        int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs);
+        if (vgNumCur == 0) break;
+        for (int32_t k = 0; k < vgNumCur; k++) {
+          SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, k);
+          sprintf(vgKey, "%s:%d", topic.topicName, pVgCur->vgId);
+          tscDebug("epoch %d vg %d build %s\n", epoch, pVgCur->vgId, vgKey);
+          taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t));
+        }
+        break;
+      }
+    }
+
+    int32_t vgNumGet = taosArrayGetSize(pTopicEp->vgs);
+    topic.vgs = taosArrayInit(vgNumGet, sizeof(SMqClientVg));
+    for (int32_t j = 0; j < vgNumGet; j++) {
       SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j);
+      sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId);
+      int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey));
+      int64_t  offset = pVgEp->offset;
+      tscDebug("epoch %d vg %d offset og to %ld\n", epoch, pVgEp->vgId, offset);
+      if (pOffset != NULL) {
+        offset = *pOffset;
+        tscDebug("epoch %d vg %d found %s\n", epoch, pVgEp->vgId, vgKey);
+      }
+      tscDebug("epoch %d vg %d offset set to %ld\n", epoch, pVgEp->vgId, offset);
       SMqClientVg clientVg = {
           .pollCnt = 0,
-          .currentOffset = pVgEp->offset,
+          .currentOffset = offset,
           .vgId = pVgEp->vgId,
           .epSet = pVgEp->epSet,
           .vgStatus = TMQ_VG_STATUS__IDLE,
@@ -922,6 +975,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
     taosArrayPush(newTopics, &topic);
   }
   if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics);
+  taosHashCleanup(pHash);
   tmq->clientTopics = newTopics;
   atomic_store_32(&tmq->epoch, epoch);
   return set;
@@ -930,8 +984,9 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
 int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
   SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param;
   tmq_t*           tmq = pParam->tmq;
+  tscDebug("consumer %ld recv ep", tmq->consumerId);
   if (code != 0) {
-    printf("get topic endpoint error, not ready, wait:%d\n", pParam->sync);
+    tscError("get topic endpoint error, not ready, wait:%d\n", pParam->sync);
     goto END;
   }
 
@@ -968,6 +1023,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
   }
 
 END:
+  atomic_store_8(&tmq->epStatus, 0);
   if (pParam->sync) {
     tsem_post(&pParam->rspSem);
   }
@@ -975,10 +1031,16 @@ END:
 }
 
 int32_t tmqAskEp(tmq_t* tmq, bool sync) {
+  int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1);
+  if (epStatus == 1) {
+    tscDebug("consumer %ld skip ask ep", tmq->consumerId);
+    return 0;
+  }
   int32_t           tlen = sizeof(SMqCMGetSubEpReq);
   SMqCMGetSubEpReq* req = taosMemoryMalloc(tlen);
   if (req == NULL) {
     tscError("failed to malloc get subscribe ep buf");
+    atomic_store_8(&tmq->epStatus, 0);
     return -1;
   }
   req->consumerId = htobe64(tmq->consumerId);
@@ -989,6 +1051,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
   if (pParam == NULL) {
     tscError("failed to malloc subscribe param");
     taosMemoryFree(req);
+    atomic_store_8(&tmq->epStatus, 0);
     return -1;
   }
   pParam->tmq = tmq;
@@ -1000,6 +1063,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
     tsem_destroy(&pParam->rspSem);
     taosMemoryFree(pParam);
     taosMemoryFree(req);
+    atomic_store_8(&tmq->epStatus, 0);
     return -1;
   }
 
@@ -1017,6 +1081,8 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
 
   SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
 
+  tscDebug("consumer %ld ask ep", tmq->consumerId);
+
   int64_t transporterId = 0;
   asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
 
@@ -1155,6 +1221,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
       SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
       int32_t      vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
       if (vgStatus != TMQ_VG_STATUS__IDLE) {
+        tscDebug("consumer %ld skip vg %d", tmq->consumerId, pVg->vgId);
         continue;
       }
       SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
@@ -1198,6 +1265,8 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
       int64_t transporterId = 0;
       /*printf("send poll\n");*/
       atomic_add_fetch_32(&tmq->waitingRequest, 1);
+      tscDebug("consumer %ld send poll: vg %d, req offset %ld", tmq->consumerId, pVg->vgId, pVg->currentOffset);
+      /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/
       asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
       pVg->pollCnt++;
       tmq->pollCnt++;
@@ -1207,13 +1276,13 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
 }
 
 // return
-int32_t tmqHandleRes(tmq_t* tmq, SMqRspHead* rspHead, bool* pReset) {
+int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspHead* rspHead, bool* pReset) {
   if (rspHead->mqMsgType == TMQ_MSG_TYPE__EP_RSP) {
     /*printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);*/
     if (rspHead->epoch > atomic_load_32(&tmq->epoch)) {
       SMqCMGetSubEpRsp* rspMsg = (SMqCMGetSubEpRsp*)rspHead;
       tmqUpdateEp(tmq, rspHead->epoch, rspMsg);
-      tmqClearUnhandleMsg(tmq);
+      /*tmqClearUnhandleMsg(tmq);*/
       *pReset = true;
     } else {
       *pReset = false;
@@ -1244,6 +1313,11 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese
         /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/
         pVg->currentOffset = rspMsg->msg.rspOffset;
         atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
+        if (rspMsg->msg.numOfTopics == 0) {
+          taosFreeQitem(rspMsg);
+          rspHead = NULL;
+          continue;
+        }
         return rspMsg;
       } else {
         /*printf("epoch mismatch\n");*/
@@ -1252,10 +1326,10 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese
     } else {
       /*printf("handle ep rsp %d\n", rspMsg->head.mqMsgType);*/
       bool reset = false;
-      tmqHandleRes(tmq, rspHead, &reset);
+      tmqHandleNoPollRsp(tmq, rspHead, &reset);
       taosFreeQitem(rspHead);
       if (pollIfReset && reset) {
-        printf("reset and repoll\n");
+        tscDebug("consumer %ld reset and repoll", tmq->consumerId);
         tmqPollImpl(tmq, blockingTime);
       }
     }
@@ -1302,6 +1376,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
 
   while (1) {
     /*printf("cycle\n");*/
+    tmqAskEp(tmq, false);
     tmqPollImpl(tmq, blocking_time);
 
     tsem_wait(&tmq->rspSem);
@@ -1493,24 +1568,3 @@ TAOS_ROW tmq_get_row(tmq_message_t* message) {
 }
 
 char* tmq_get_topic_name(tmq_message_t* message) { return "not implemented yet"; }
-
-#if 0
-tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) {
-  tmq_t* pTmq = taosMemoryMalloc(sizeof(tmq_t));
-  if (pTmq == NULL) {
-    return NULL;
-  }
-  strcpy(pTmq->groupId, conf->groupId);
-  strcpy(pTmq->clientId, conf->clientId);
-  pTmq->pTscObj = (STscObj*)conn;
-  pTmq->pTscObj->connType = HEARTBEAT_TYPE_MQ;
-  return pTmq;
-}
-
-
-static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) {
-  assert(pMsgBody != NULL);
-  taosMemoryFreeClear(pMsgBody->msgInfo.pData);
-  taosMemoryFreeClear(pMsgBody);
-}
-#endif

source/common/src/tdatablock.c

@@ -331,7 +331,6 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock) {
     return 0;
   }
 
-  ASSERT(pColInfoData->nullbitmap == NULL);
   pDataBlock->info.window.skey = *(TSKEY*)colDataGetData(pColInfoData, 0);
   pDataBlock->info.window.ekey = *(TSKEY*)colDataGetData(pColInfoData, (pDataBlock->info.rows - 1));
   return 0;
@@ -609,22 +608,6 @@ size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) {
   return sizeof(int32_t) + pBlock->info.numOfCols * sizeof(int32_t);
 }
 
-SSchema* blockDataExtractSchema(const SSDataBlock* pBlock, int32_t* numOfCols) {
-  SSchema* pSchema = taosMemoryCalloc(pBlock->info.numOfCols, sizeof(SSchema));
-  for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
-    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
-    pSchema[i].bytes = pColInfoData->info.bytes;
-    pSchema[i].type = pColInfoData->info.type;
-    pSchema[i].colId = pColInfoData->info.colId;
-  }
-
-  if (numOfCols != NULL) {
-    *numOfCols = pBlock->info.numOfCols;
-  }
-
-  return pSchema;
-}
-
 double blockDataGetSerialRowSize(const SSDataBlock* pBlock) {
   ASSERT(pBlock != NULL);
   double rowSize = 0;
@@ -1156,7 +1139,7 @@ void blockDataCleanup(SSDataBlock* pDataBlock) {
   }
 }
 
-int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) {
+int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) {
   if (0 == numOfRows) {
     return TSDB_CODE_SUCCESS;
   }
@@ -1198,7 +1181,7 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
 
   for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) {
     SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
-    code = blockDataEnsureColumnCapacity(p, numOfRows);
+    code = colInfoDataEnsureCapacity(p, numOfRows);
     if (code) {
       return code;
     }
@@ -1250,6 +1233,63 @@ void colDataDestroy(SColumnInfoData* pColData) {
   taosMemoryFree(pColData->pData);
 }
 
+
+static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) {
+  int32_t len = BitmapLen(total);
+
+  int32_t newLen = BitmapLen(total - n);
+  if (n%8 == 0) {
+    memmove(nullBitmap, nullBitmap + n/8, newLen);
+  } else {
+    int32_t tail = n % 8;
+    int32_t i = 0;
+
+    uint8_t* p = (uint8_t*) nullBitmap;
+    while(i < len) {
+      uint8_t v = p[i];
+
+      p[i] = 0;
+      p[i] = (v << tail);
+
+      if (i < len - 1) {
+        uint8_t next = p[i + 1];
+        p[i] |= (next >> (8 - tail));
+      }
+
+      i += 1;
+    }
+  }
+}
+
+static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) {
+  if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
+    memmove(pColInfoData->varmeta.offset, &pColInfoData->varmeta.offset[n], (total - n));
+    memset(&pColInfoData->varmeta.offset[total - n - 1], 0, n);
+  } else {
+    int32_t bytes = pColInfoData->info.bytes;
+    memmove(pColInfoData->pData, ((char*)pColInfoData->pData + n * bytes), (total - n) * bytes);
+    doShiftBitmap(pColInfoData->nullbitmap, n, total);
+  }
+}
+
+int32_t blockDataTrimFirstNRows(SSDataBlock *pBlock, size_t n) {
+  if (n == 0) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (pBlock->info.rows <= n) {
+    blockDataCleanup(pBlock);
+  } else {
+    for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
+      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
+      colDataTrimFirstNRows(pColInfoData, n, pBlock->info.rows);
+    }
+
+    pBlock->info.rows -= n;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
   int64_t tbUid = pBlock->info.uid;
   int16_t numOfCols = pBlock->info.numOfCols;
@@ -1389,6 +1429,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
 
   return buf;
 }
+
 void blockDebugShowData(const SArray* dataBlocks) {
   char    pBuf[128];
   int32_t sz = taosArrayGetSize(dataBlocks);

source/common/src/tdataformat.c

@@ -85,6 +85,7 @@ int tdEncodeSchema(void **buf, STSchema *pSchema) {
   for (int i = 0; i < schemaNCols(pSchema); i++) {
     STColumn *pCol = schemaColAt(pSchema, i);
     tlen += taosEncodeFixedI8(buf, colType(pCol));
+    tlen += taosEncodeFixedI8(buf, colSma(pCol));
     tlen += taosEncodeFixedI16(buf, colColId(pCol));
     tlen += taosEncodeFixedI16(buf, colBytes(pCol));
   }
@@ -107,12 +108,14 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) {
 
   for (int i = 0; i < numOfCols; i++) {
     col_type_t  type = 0;
+    int8_t      sma = TSDB_BSMA_TYPE_NONE;
     col_id_t    colId = 0;
     col_bytes_t bytes = 0;
     buf = taosDecodeFixedI8(buf, &type);
+    buf = taosDecodeFixedI8(buf, &sma);
     buf = taosDecodeFixedI16(buf, &colId);
     buf = taosDecodeFixedI32(buf, &bytes);
-    if (tdAddColToSchema(&schemaBuilder, type, colId, bytes) < 0) {
+    if (tdAddColToSchema(&schemaBuilder, type, sma, colId, bytes) < 0) {
       tdDestroyTSchemaBuilder(&schemaBuilder);
       return NULL;
     }
@@ -148,7 +151,7 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) {
   pBuilder->version = version;
 }
 
-int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col_bytes_t bytes) {
+int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t sma, col_id_t colId, col_bytes_t bytes) {
   if (!isValidDataType(type)) return -1;
 
   if (pBuilder->nCols >= pBuilder->tCols) {
@@ -161,6 +164,7 @@ int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col
   STColumn *pCol = &(pBuilder->columns[pBuilder->nCols]);
   colSetType(pCol, type);
   colSetColId(pCol, colId);
+  colSetSma(pCol, sma);
   if (pBuilder->nCols == 0) {
     colSetOffset(pCol, 0);
   } else {
@@ -168,9 +172,6 @@ int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, col_id_t colId, col
     colSetOffset(pCol, pTCol->offset + TYPE_BYTES[pTCol->type]);
   }
 
-  // TODO: set sma value by user input
-  pCol->sma = 1;
-
   if (IS_VAR_DATA_TYPE(type)) {
     colSetBytes(pCol, bytes);
     pBuilder->tlen += (TYPE_BYTES[type] + bytes);

source/common/src/tglobal.c

@@ -45,7 +45,7 @@ float   tsRatioOfQueryCores = 1.0f;
 int32_t tsMaxBinaryDisplayWidth = 30;
 bool    tsEnableSlaveQuery = 1;
 bool    tsPrintAuth = 0;
-int32_t tsMultiProcess = 0;
+bool    tsMultiProcess = 0;
 
 // monitor
 bool     tsEnableMonitor = 1;
@@ -347,7 +347,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, 0) != 0) return -1;
   if (cfgAddBool(pCfg, "slaveQuery", tsEnableSlaveQuery, 0) != 0) return -1;
   if (cfgAddBool(pCfg, "deadLockKillQuery", tsDeadLockKillQuery, 0) != 0) return -1;
-  if (cfgAddInt32(pCfg, "multiProcess", tsMultiProcess, 0, 2, 0) != 0) return -1;
+  if (cfgAddBool(pCfg, "multiProcess", tsMultiProcess, 0) != 0) return -1;
 
   if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 360000, 0) != 0) return -1;
@@ -466,7 +466,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsPrintAuth = cfgGetItem(pCfg, "printAuth")->bval;
   tsEnableSlaveQuery = cfgGetItem(pCfg, "slaveQuery")->bval;
   tsDeadLockKillQuery = cfgGetItem(pCfg, "deadLockKillQuery")->bval;
-  tsMultiProcess = cfgGetItem(pCfg, "multiProcess")->i32;
+  tsMultiProcess = cfgGetItem(pCfg, "multiProcess")->bval;
 
   tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval;
   tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32;

source/common/src/tmsg.c

@@ -296,24 +296,22 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
   switch (pReq->type) {
     case TD_SUPER_TABLE:
       tlen += taosEncodeFixedI64(buf, pReq->stbCfg.suid);
-      tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nCols);
-      for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) {
+      tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nCols);
+      tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nBSmaCols);
+      for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) {
         tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].type);
+        tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].sma);
         tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pSchema[i].colId);
         tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].bytes);
         tlen += taosEncodeString(buf, pReq->stbCfg.pSchema[i].name);
       }
-      tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nTagCols);
-      for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) {
+      tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nTagCols);
+      for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) {
         tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].type);
         tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pTagSchema[i].colId);
         tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes);
         tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name);
       }
-      tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nBSmaCols);
-      for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) {
-        tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pBSmaCols[i]);
-      }
       if (pReq->rollup && pReq->stbCfg.pRSmaParam) {
         SRSmaParam *param = pReq->stbCfg.pRSmaParam;
         tlen += taosEncodeFixedU32(buf, (uint32_t)param->xFilesFactor);
@@ -330,17 +328,15 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
       tlen += tdEncodeKVRow(buf, pReq->ctbCfg.pTag);
       break;
     case TD_NORMAL_TABLE:
-      tlen += taosEncodeFixedU32(buf, pReq->ntbCfg.nCols);
-      for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) {
+      tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nCols);
+      tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nBSmaCols);
+      for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) {
         tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].type);
+        tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].sma);
         tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pSchema[i].colId);
         tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes);
         tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name);
       }
-      tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nBSmaCols);
-      for (col_id_t i = 0; i < pReq->ntbCfg.nBSmaCols; ++i) {
-        tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pBSmaCols[i]);
-      }
       if (pReq->rollup && pReq->ntbCfg.pRSmaParam) {
         SRSmaParam *param = pReq->ntbCfg.pRSmaParam;
         tlen += taosEncodeFixedU32(buf, (uint32_t)param->xFilesFactor);
@@ -370,31 +366,24 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
   switch (pReq->type) {
     case TD_SUPER_TABLE:
       buf = taosDecodeFixedI64(buf, &(pReq->stbCfg.suid));
-      buf = taosDecodeFixedU32(buf, &(pReq->stbCfg.nCols));
-      pReq->stbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchema));
-      for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) {
+      buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nCols));
+      buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols));
+      pReq->stbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchemaEx));
+      for (col_id_t i = 0; i < pReq->stbCfg.nCols; i++) {
         buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type));
+        buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].sma));
         buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.pSchema[i].colId));
         buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].bytes));
         buf = taosDecodeStringTo(buf, pReq->stbCfg.pSchema[i].name);
       }
-      buf = taosDecodeFixedU32(buf, &pReq->stbCfg.nTagCols);
+      buf = taosDecodeFixedI16(buf, &pReq->stbCfg.nTagCols);
       pReq->stbCfg.pTagSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nTagCols * sizeof(SSchema));
-      for (uint32_t i = 0; i < pReq->stbCfg.nTagCols; i++) {
+      for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; i++) {
         buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type));
         buf = taosDecodeFixedI16(buf, &pReq->stbCfg.pTagSchema[i].colId);
         buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes);
         buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name);
       }
-      buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols));
-      if (pReq->stbCfg.nBSmaCols > 0) {
-        pReq->stbCfg.pBSmaCols = (col_id_t *)taosMemoryMalloc(pReq->stbCfg.nBSmaCols * sizeof(col_id_t));
-        for (col_id_t i = 0; i < pReq->stbCfg.nBSmaCols; ++i) {
-          buf = taosDecodeFixedI16(buf, pReq->stbCfg.pBSmaCols + i);
-        }
-      } else {
-        pReq->stbCfg.pBSmaCols = NULL;
-      }
       if (pReq->rollup) {
         pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam));
         SRSmaParam *param = pReq->stbCfg.pRSmaParam;
@@ -418,23 +407,16 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
       buf = tdDecodeKVRow(buf, &pReq->ctbCfg.pTag);
       break;
     case TD_NORMAL_TABLE:
-      buf = taosDecodeFixedU32(buf, &pReq->ntbCfg.nCols);
-      pReq->ntbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchema));
-      for (uint32_t i = 0; i < pReq->ntbCfg.nCols; i++) {
+      buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.nCols);
+      buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols));
+      pReq->ntbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchemaEx));
+      for (col_id_t i = 0; i < pReq->ntbCfg.nCols; i++) {
         buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type);
+        buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].sma);
         buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.pSchema[i].colId);
         buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes);
         buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name);
       }
-      buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols));
-      if (pReq->ntbCfg.nBSmaCols > 0) {
-        pReq->ntbCfg.pBSmaCols = (col_id_t *)taosMemoryMalloc(pReq->ntbCfg.nBSmaCols * sizeof(col_id_t));
-        for (col_id_t i = 0; i < pReq->ntbCfg.nBSmaCols; ++i) {
-          buf = taosDecodeFixedI16(buf, pReq->ntbCfg.pBSmaCols + i);
-        }
-      } else {
-        pReq->ntbCfg.pBSmaCols = NULL;
-      }
       if (pReq->rollup) {
         pReq->ntbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam));
         SRSmaParam *param = pReq->ntbCfg.pRSmaParam;
@@ -1629,6 +1611,7 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
   if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->quorum) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->replications) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -1650,6 +1633,7 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
   if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->quorum) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->replications) < 0) return -1;
   tEndDecode(&decoder);
 
   tCoderClear(&decoder);

source/common/src/ttime.c

@@ -361,6 +361,18 @@ int32_t parseLocaltimeDst(char* timestr, int64_t* time, int32_t timePrec) {
   return 0;
 }
 
+char getPrecisionUnit(int32_t precision) {
+  static char units[3] = {TIME_UNIT_MILLISECOND, TIME_UNIT_MICROSECOND, TIME_UNIT_NANOSECOND};
+  switch (precision) {
+    case TSDB_TIME_PRECISION_MILLI:
+    case TSDB_TIME_PRECISION_MICRO:
+    case TSDB_TIME_PRECISION_NANO:
+      return units[precision];
+    default:
+      return 0;
+  }
+}
+
 int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision) {
   assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO ||
          fromPrecision == TSDB_TIME_PRECISION_NANO);
@@ -370,6 +382,33 @@ int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrec
   return (int64_t)((double)time * factors[fromPrecision][toPrecision]);
 }
 
+int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit) {
+  assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO ||
+         fromPrecision == TSDB_TIME_PRECISION_NANO);
+  static double factors[3] = {1000000., 1000., 1.};
+  switch (toUnit) {
+    case 's':
+      return time * factors[fromPrecision] / NANOSECOND_PER_SEC;
+    case 'm':
+      return time * factors[fromPrecision] / NANOSECOND_PER_MINUTE;
+    case 'h':
+      return time * factors[fromPrecision] / NANOSECOND_PER_HOUR;
+    case 'd':
+      return time * factors[fromPrecision] / NANOSECOND_PER_DAY;
+    case 'w':
+      return time * factors[fromPrecision] / NANOSECOND_PER_WEEK;
+    case 'a':
+      return time * factors[fromPrecision] / NANOSECOND_PER_MSEC;
+    case 'u':
+      return time * factors[fromPrecision] / NANOSECOND_PER_USEC;
+    case 'b':
+      return time * factors[fromPrecision];
+    default: {
+      return -1;
+    }
+  }  
+}
+
 static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) {
   switch (unit) {
     case 's':
@@ -688,4 +727,4 @@ void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t t, int32_t precision)
   length += (int32_t)strftime(ts + length, 40 - length, "%z", ptm);
 
   tstrncpy(buf, ts, bufLen);
-}
+}
\ No newline at end of file

source/common/src/ttypes.c

@@ -431,7 +431,7 @@ FORCE_INLINE void *getDataMax(int32_t type) {
   }
 }
 
-bool isValidDataType(int32_t type) { return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_UBIGINT; }
+bool isValidDataType(int32_t type) { return type >= TSDB_DATA_TYPE_NULL && type < TSDB_DATA_TYPE_MAX; }
 
 void setVardataNull(void *val, int32_t type) {
   if (type == TSDB_DATA_TYPE_BINARY) {

source/dnode/mgmt/CMakeLists.txt

-aux_source_directory(src DNODE_SRC)
-aux_source_directory(dnode/src DNODE_SRC)
-aux_source_directory(qnode/src DNODE_SRC)
-aux_source_directory(bnode/src DNODE_SRC)
-aux_source_directory(snode/src DNODE_SRC)
-aux_source_directory(vnode/src DNODE_SRC)
-aux_source_directory(mnode/src DNODE_SRC)
-aux_source_directory(container/src DNODE_SRC)
-
+aux_source_directory(dm/src DNODE_SRC)
+aux_source_directory(qm/src DNODE_SRC)
+aux_source_directory(bm/src DNODE_SRC)
+aux_source_directory(sm/src DNODE_SRC)
+aux_source_directory(vm/src DNODE_SRC)
+aux_source_directory(mm/src DNODE_SRC)
+aux_source_directory(main/src DNODE_SRC)
 add_library(dnode STATIC ${DNODE_SRC})
 target_link_libraries(
     dnode cjson mnode vnode qnode snode bnode wal sync taos tfs monitor
@@ -14,16 +12,22 @@ target_link_libraries(
 target_include_directories(
     dnode
     PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mgmt"
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/dnode/inc"
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/qnode/inc"
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/bnode/inc"
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/snode/inc"
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/vnode/inc"
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/mnode/inc"
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/container/inc"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/dm/inc"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/qm/inc"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/bm/inc"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/sm/inc"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/vm/inc"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/mm/inc"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/main/inc"
 )
 
-add_subdirectory(main)
+aux_source_directory(main/exe EXEC_SRC)
+add_executable(taosd ${EXEC_SRC})
+target_include_directories(
+    taosd
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/main/inc"
+)
+target_link_libraries(taosd dnode)
 
 if(${BUILD_TEST})
     add_subdirectory(test)

source/dnode/mgmt/container/src/dndFile.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http:www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dndInt.h"
-
-#define MAXLEN 1024
-
-int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed) {
-  int32_t       code = TSDB_CODE_NODE_PARSE_FILE_ERROR;
-  int32_t       len = 0;
-  const int32_t maxLen = MAXLEN;
-  char          content[MAXLEN + 1] = {0};
-  cJSON        *root = NULL;
-  char          file[PATH_MAX];
-  TdFilePtr     pFile = NULL;
-
-  snprintf(file, sizeof(file), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name);
-  pFile = taosOpenFile(file, TD_FILE_READ);
-  if (pFile == NULL) {
-    dDebug("file %s not exist", file);
-    code = 0;
-    goto _OVER;
-  }
-
-  len = (int32_t)taosReadFile(pFile, content, maxLen);
-  if (len <= 0) {
-    dError("failed to read %s since content is null", file);
-    goto _OVER;
-  }
-
-  content[len] = 0;
-  root = cJSON_Parse(content);
-  if (root == NULL) {
-    dError("failed to read %s since invalid json format", file);
-    goto _OVER;
-  }
-
-  cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
-  if (!deployed || deployed->type != cJSON_Number) {
-    dError("failed to read %s since deployed not found", file);
-    goto _OVER;
-  }
-  *pDeployed = deployed->valueint != 0;
-
-  code = 0;
-  dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed);
-
-_OVER:
-  if (root != NULL) cJSON_Delete(root);
-  if (pFile != NULL) taosCloseFile(&pFile);
-
-  terrno = code;
-  return code;
-}
-
-int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed) {
-  char file[PATH_MAX] = {0};
-  snprintf(file, sizeof(file), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name);
-
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-  if (pFile == NULL) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    dError("failed to write %s since %s", file, terrstr());
-    return -1;
-  }
-
-  int32_t       len = 0;
-  const int32_t maxLen = MAXLEN;
-  char          content[MAXLEN + 1] = {0};
-
-  len += snprintf(content + len, maxLen - len, "{\n");
-  len += snprintf(content + len, maxLen - len, "  \"deployed\": %d\n", deployed);
-  len += snprintf(content + len, maxLen - len, "}\n");
-
-  taosWriteFile(pFile, content, len);
-  taosFsyncFile(pFile);
-  taosCloseFile(&pFile);
-
-  char realfile[PATH_MAX] = {0};
-  snprintf(realfile, sizeof(realfile), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name);
-
-  if (taosRenameFile(file, realfile) != 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    dError("failed to rename %s since %s", file, terrstr());
-    return -1;
-  }
-
-  dInfo("successed to write %s, deployed:%d", realfile, deployed);
-  return 0;
-}

source/dnode/mgmt/dnode/src/dmFile.c → source/dnode/mgmt/dm/src/dmFile.c

@@ -130,7 +130,7 @@ int32_t dmReadFile(SDnodeMgmt *pMgmt) {
   }
 
   code = 0;
-  dInfo("succcessed to read file %s", file);
+  dDebug("succcessed to read file %s", file);
   dmPrintDnodes(pMgmt);
 
 PRASE_DNODE_OVER:
@@ -200,7 +200,7 @@ int32_t dmWriteFile(SDnodeMgmt *pMgmt) {
   taosMemoryFree(content);
 
   char realfile[PATH_MAX];
-  snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP);
+  snprintf(realfile, sizeof(realfile), "%s%sdnode.json", pMgmt->path, TD_DIRSEP);
 
   if (taosRenameFile(file, realfile) != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);

source/dnode/mgmt/dnode/src/dmInt.c → source/dnode/mgmt/dm/src/dmInt.c

@@ -112,6 +112,16 @@ int32_t dmInit(SMgmtWrapper *pWrapper) {
     return -1;
   }
 
+  if (dndInitServer(pDnode) != 0) {
+    dError("failed to init trans server since %s", terrstr());
+    return -1;
+  }
+
+  if (dndInitClient(pDnode) != 0) {
+    dError("failed to init trans client since %s", terrstr());
+    return -1;
+  }
+
   pWrapper->pMgmt = pMgmt;
   dInfo("dnode-mgmt is initialized");
   return 0;
@@ -122,6 +132,7 @@ void dmCleanup(SMgmtWrapper *pWrapper) {
   if (pMgmt == NULL) return;
 
   dInfo("dnode-mgmt start to clean up");
+  SDnode *pDnode = pMgmt->pDnode;
   dmStopWorker(pMgmt);
 
   taosWLockLatch(&pMgmt->latch);
@@ -140,6 +151,9 @@ void dmCleanup(SMgmtWrapper *pWrapper) {
 
   taosMemoryFree(pMgmt);
   pWrapper->pMgmt = NULL;
+  dndCleanupServer(pDnode);
+  dndCleanupClient(pDnode);
+
   dInfo("dnode-mgmt is cleaned up");
 }
 

source/dnode/mgmt/dnode/src/dmMsg.c → source/dnode/mgmt/dm/src/dmMsg.c

@@ -53,7 +53,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
   tSerializeSStatusReq(pHead, contLen, &req);
   taosArrayDestroy(req.pVloads);
 
-  SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)9527};
+  SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)0x9527};
   pMgmt->statusSent = 1;
 
   dTrace("send req:%s to mnode, app:%p", TMSG_INFO(rpcMsg.msgType), rpcMsg.ahandle);

source/dnode/mgmt/main/CMakeLists.txt

-aux_source_directory(src EXEC_SRC)
-add_executable(taosd ${EXEC_SRC})
-
-target_include_directories(
-    taosd
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
-)
-
-target_link_libraries(taosd dnode)

source/dnode/mgmt/main/src/dndMain.c → source/dnode/mgmt/main/exe/dndMain.c

@@ -14,20 +14,23 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "dndMain.h"
+#include "dnd.h"
+#include "tconfig.h"
 
 static struct {
-  bool    dumpConfig;
-  bool    generateGrant;
-  bool    printAuth;
-  bool    printVersion;
-  char    envFile[PATH_MAX];
-  char    apolloUrl[PATH_MAX];
-  SDnode *pDnode;
+  bool      dumpConfig;
+  bool      generateGrant;
+  bool      printAuth;
+  bool      printVersion;
+  char      envFile[PATH_MAX];
+  char      apolloUrl[PATH_MAX];
+  SArray   *pArgs;  // SConfigPair
+  SDnode   *pDnode;
+  ENodeType ntype;
 } global = {0};
 
-static void dndSigintHandle(int signum, void *info, void *ctx) {
-  dInfo("signal:%d is received", signum);
+static void dndStopDnode(int signum, void *info, void *ctx) {
+  dInfo("system signal:%d received", signum);
   SDnode *pDnode = atomic_val_compare_exchange_ptr(&global.pDnode, 0, global.pDnode);
   if (pDnode != NULL) {
     dndHandleEvent(pDnode, DND_EVENT_STOP);
@@ -35,14 +38,21 @@ static void dndSigintHandle(int signum, void *info, void *ctx) {
 }
 
 static void dndSetSignalHandle() {
-  taosSetSignal(SIGTERM, dndSigintHandle);
-  taosSetSignal(SIGHUP, dndSigintHandle);
-  taosSetSignal(SIGINT, dndSigintHandle);
-  taosSetSignal(SIGABRT, dndSigintHandle);
-  taosSetSignal(SIGBREAK, dndSigintHandle);
+  taosSetSignal(SIGTERM, dndStopDnode);
+  taosSetSignal(SIGHUP, dndStopDnode);
+  taosSetSignal(SIGINT, dndStopDnode);
+  taosSetSignal(SIGABRT, dndStopDnode);
+  taosSetSignal(SIGBREAK, dndStopDnode);
+
+  if (!tsMultiProcess) {
+  } else if (global.ntype == DNODE || global.ntype == NODE_MAX) {
+    taosIgnSignal(SIGCHLD);
+  } else {
+    taosKillChildOnParentStopped();
+  }
 }
 
-static int32_t dndParseOption(int32_t argc, char const *argv[]) {
+static int32_t dndParseArgs(int32_t argc, char const *argv[]) {
   for (int32_t i = 1; i < argc; ++i) {
     if (strcmp(argv[i], "-c") == 0) {
       if (i < argc - 1) {
@@ -55,10 +65,20 @@ static int32_t dndParseOption(int32_t argc, char const *argv[]) {
         printf("'-c' requires a parameter, default is %s\n", configDir);
         return -1;
       }
-    } else if (strcmp(argv[i], "-C") == 0) {
-      global.dumpConfig = true;
+    } else if (strcmp(argv[i], "-a") == 0) {
+      tstrncpy(global.apolloUrl, argv[++i], PATH_MAX);
+    } else if (strcmp(argv[i], "-e") == 0) {
+      tstrncpy(global.envFile, argv[++i], PATH_MAX);
+    } else if (strcmp(argv[i], "-n") == 0) {
+      global.ntype = atoi(argv[++i]);
+      if (global.ntype <= DNODE || global.ntype > NODE_MAX) {
+        printf("'-n' range is [1-5], default is 0\n");
+        return -1;
+      }
     } else if (strcmp(argv[i], "-k") == 0) {
       global.generateGrant = true;
+    } else if (strcmp(argv[i], "-C") == 0) {
+      global.dumpConfig = true;
     } else if (strcmp(argv[i], "-V") == 0) {
       global.printVersion = true;
     } else {
@@ -68,26 +88,77 @@ static int32_t dndParseOption(int32_t argc, char const *argv[]) {
   return 0;
 }
 
+static void dndGenerateGrant() {
+  // grantParseParameter();
+  printf("this feature is not implemented yet\n");
+}
+
+static void dndPrintVersion() {
+#ifdef TD_ENTERPRISE
+  char *releaseName = "enterprise";
+#else
+  char *releaseName = "community";
+#endif
+  printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version);
+  printf("gitinfo: %s\n", gitinfo);
+  printf("buildInfo: %s\n", buildinfo);
+}
+
+static void dndDumpCfg() {
+  SConfig *pCfg = taosGetCfg();
+  cfgDumpCfg(pCfg, 0, 1);
+}
+
+static SDnodeOpt dndGetOpt() {
+  SConfig  *pCfg = taosGetCfg();
+  SDnodeOpt option = {0};
+
+  option.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32;
+  tstrncpy(option.dataDir, tsDataDir, sizeof(option.dataDir));
+  tstrncpy(option.firstEp, tsFirst, sizeof(option.firstEp));
+  tstrncpy(option.secondEp, tsSecond, sizeof(option.firstEp));
+  option.serverPort = tsServerPort;
+  tstrncpy(option.localFqdn, tsLocalFqdn, sizeof(option.localFqdn));
+  snprintf(option.localEp, sizeof(option.localEp), "%s:%u", option.localFqdn, option.serverPort);
+  option.disks = tsDiskCfg;
+  option.numOfDisks = tsDiskCfgNum;
+  option.ntype = global.ntype;
+  return option;
+}
+
+static int32_t dndInitLog() {
+  char logName[12] = {0};
+  snprintf(logName, sizeof(logName), "%slog", dndNodeLogStr(global.ntype));
+  return taosCreateLog(logName, 1, configDir, global.envFile, global.apolloUrl, global.pArgs, 0);
+}
+
+static void dndSetProcInfo(int32_t argc, char **argv) {
+  taosSetProcPath(argc, argv);
+  if (global.ntype != DNODE && global.ntype != NODE_MAX) {
+    const char *name = dndNodeProcStr(global.ntype);
+    taosSetProcName(argc, argv, name);
+  }
+}
+
 static int32_t dndRunDnode() {
   if (dndInit() != 0) {
-    dInfo("failed to initialize dnode environment since %s", terrstr());
+    dError("failed to init environment since %s", terrstr());
     return -1;
   }
 
   SDnodeOpt option = dndGetOpt();
-
-  SDnode *pDnode = dndCreate(&option);
+  SDnode   *pDnode = dndCreate(&option);
   if (pDnode == NULL) {
-    dError("failed to to create dnode object since %s", terrstr());
+    dError("failed to to create dnode since %s", terrstr());
     return -1;
   } else {
     global.pDnode = pDnode;
     dndSetSignalHandle();
   }
 
-  dInfo("start the TDengine service");
+  dInfo("start the service");
   int32_t code = dndRun(pDnode);
-  dInfo("start shutting down the TDengine service");
+  dInfo("start shutting down the service");
 
   global.pDnode = NULL;
   dndClose(pDnode);
@@ -99,11 +170,12 @@ static int32_t dndRunDnode() {
 
 int main(int argc, char const *argv[]) {
   if (!taosCheckSystemIsSmallEnd()) {
-    dError("failed to start TDengine since on non-small-end machines");
+    printf("failed to start since on non-small-end machines\n");
     return -1;
   }
 
-  if (dndParseOption(argc, argv) != 0) {
+  if (dndParseArgs(argc, argv) != 0) {
+    printf("failed to start since parse args error\n");
     return -1;
   }
 
@@ -117,13 +189,13 @@ int main(int argc, char const *argv[]) {
     return 0;
   }
 
-  if (taosCreateLog("taosdlog", 1, configDir, global.envFile, global.apolloUrl, NULL, 0) != 0) {
-    dError("failed to start TDengine since read log config error");
+  if (dndInitLog() != 0) {
+    printf("failed to start since init log error\n");
     return -1;
   }
 
-  if (taosInitCfg(configDir, global.envFile, global.apolloUrl, NULL, 0) != 0) {
-    dError("failed to start TDengine since read config error");
+  if (taosInitCfg(configDir, global.envFile, global.apolloUrl, global.pArgs, 0) != 0) {
+    dError("failed to start since read config error");
     return -1;
   }
 
@@ -134,5 +206,6 @@ int main(int argc, char const *argv[]) {
     return 0;
   }
 
+  dndSetProcInfo(argc, (char **)argv);
   return dndRunDnode();
 }

source/dnode/mgmt/container/inc/dnd.h → source/dnode/mgmt/main/inc/dnd.h

@@ -95,12 +95,14 @@ typedef struct SMgmtWrapper {
   bool        deployed;
   bool        required;
   EProcType   procType;
+  int32_t     procId;
   SProcObj   *pProc;
+  SShm        shm;
   void       *pMgmt;
   SDnode     *pDnode;
-  NodeMsgFp   msgFps[TDMT_MAX];
-  int32_t     msgVgIds[TDMT_MAX];  // Handle the case where the same message type is distributed to qnode or vnode
   SMgmtFp     fp;
+  int8_t      msgVgIds[TDMT_MAX];  // Handle the case where the same message type is distributed to qnode or vnode
+  NodeMsgFp   msgFps[TDMT_MAX];
 } SMgmtWrapper;
 
 typedef struct {
@@ -119,31 +121,39 @@ typedef struct SDnode {
   char        *firstEp;
   char        *secondEp;
   char        *dataDir;
-  SDiskCfg    *pDisks;
+  SDiskCfg    *disks;
   int32_t      numOfDisks;
   uint16_t     serverPort;
   bool         dropped;
+  ENodeType    ntype;
   EDndStatus   status;
   EDndEvent    event;
   SStartupReq  startup;
-  TdFilePtr    pLockFile;
+  TdFilePtr    lockfile;
   STransMgmt   trans;
   SMgmtWrapper wrappers[NODE_MAX];
 } SDnode;
 
-EDndStatus dndGetStatus(SDnode *pDnode);
-void       dndSetStatus(SDnode *pDnode, EDndStatus stat);
-void       dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp, int32_t vgId);
-void       dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc);
-void       dndSendMonitorReport(SDnode *pDnode);
-
+const char *dndNodeLogStr(ENodeType ntype);
+const char *dndNodeProcStr(ENodeType ntype);
+const char *dndEventStr(EDndEvent ev);
+EDndStatus  dndGetStatus(SDnode *pDnode);
+void        dndSetStatus(SDnode *pDnode, EDndStatus stat);
+void        dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId);
+void        dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc);
+void        dndSendMonitorReport(SDnode *pDnode);
+
+int32_t dndInitServer(SDnode *pDnode);
+void    dndCleanupServer(SDnode *pDnode);
+int32_t dndInitClient(SDnode *pDnode);
+void    dndCleanupClient(SDnode *pDnode);
+int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg);
 int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
 int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pMsg);
 void    dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp);
 void    dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
 SMsgCb  dndCreateMsgcb(SMgmtWrapper *pWrapper);
 
-int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg);
 int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed);
 int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed);
 

source/dnode/mgmt/container/inc/dndInt.h → source/dnode/mgmt/main/inc/dndInt.h

@@ -34,7 +34,6 @@ int32_t     dndInit();
 void        dndCleanup();
 const char *dndStatStr(EDndStatus stat);
 void        dndGetStartup(SDnode *pDnode, SStartupReq *pStartup);
-TdFilePtr   dndCheckRunning(const char *dataDir);
 void        dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pMsg);
 
 // dndMsg.c
@@ -51,13 +50,14 @@ void    dndClose(SDnode *pDnode);
 void    dndHandleEvent(SDnode *pDnode, EDndEvent event);
 
 // dndTransport.c
-int32_t dndInitServer(SDnode *pDnode);
-void    dndCleanupServer(SDnode *pDnode);
-int32_t dndInitClient(SDnode *pDnode);
-void    dndCleanupClient(SDnode *pDnode);
 int32_t dndInitMsgHandle(SDnode *pDnode);
 void    dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp);
 
+// dndFile.c
+TdFilePtr dndCheckRunning(const char *dataDir);
+int32_t   dndReadShmFile(SDnode *pDnode);
+int32_t   dndWriteShmFile(SDnode *pDnode);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mgmt/container/src/dndInt.c → source/dnode/mgmt/main/src/dndEnv.c

@@ -20,7 +20,7 @@
 static int8_t once = DND_ENV_INIT;
 
 int32_t dndInit() {
-  dInfo("start to init dnode env");
+  dDebug("start to init dnode env");
   if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) {
     terrno = TSDB_CODE_REPEAT_INIT;
     dError("failed to init dnode env since %s", terrstr());
@@ -31,12 +31,6 @@ int32_t dndInit() {
   taosBlockSIGPIPE();
   taosResolveCRC();
 
-  if (rpcInit() != 0) {
-    dError("failed to init rpc since %s", terrstr());
-    dndCleanup();
-    return -1;
-  }
-
   SMonCfg monCfg = {0};
   monCfg.maxLogs = tsMonitorMaxLogs;
   monCfg.port = tsMonitorPort;
@@ -44,7 +38,6 @@ int32_t dndInit() {
   monCfg.comp = tsMonitorComp;
   if (monInit(&monCfg) != 0) {
     dError("failed to init monitor since %s", terrstr());
-    dndCleanup();
     return -1;
   }
 
@@ -53,20 +46,19 @@ int32_t dndInit() {
 }
 
 void dndCleanup() {
-  dInfo("start to cleanup dnode env");
+  dDebug("start to cleanup dnode env");
   if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) {
     dError("dnode env is already cleaned up");
     return;
   }
 
   monCleanup();
-  rpcCleanup();
   walCleanUp();
   taosStopCacheRefreshWorker();
   dInfo("dnode env is cleaned up");
 }
 
-void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp, int32_t vgId) {
+void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId) {
   pWrapper->msgFps[TMSG_INDEX(msgType)] = nodeMsgFp;
   pWrapper->msgVgIds[TMSG_INDEX(msgType)] = vgId;
 }
@@ -80,19 +72,6 @@ void dndSetStatus(SDnode *pDnode, EDndStatus status) {
   }
 }
 
-const char *dndStatStr(EDndStatus status) {
-  switch (status) {
-    case DND_STAT_INIT:
-      return "init";
-    case DND_STAT_RUNNING:
-      return "running";
-    case DND_STAT_STOPPED:
-      return "stopped";
-    default:
-      return "unknown";
-  }
-}
-
 void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) {
   SStartupReq *pStartup = &pDnode->startup;
   tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN);
@@ -105,29 +84,6 @@ void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) {
   pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING);
 }
 
-TdFilePtr dndCheckRunning(const char *dataDir) {
-  char filepath[PATH_MAX] = {0};
-  snprintf(filepath, sizeof(filepath), "%s/.running", dataDir);
-
-  TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
-  if (pFile == NULL) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    dError("failed to lock file:%s since %s", filepath, terrstr());
-    return NULL;
-  }
-
-  int32_t ret = taosLockFile(pFile);
-  if (ret != 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    dError("failed to lock file:%s since %s", filepath, terrstr());
-    taosCloseFile(&pFile);
-    return NULL;
-  }
-
-  dDebug("file:%s is locked", filepath);
-  return pFile;
-}
-
 void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) {
   dDebug("startup req is received");
   SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq));

source/dnode/mgmt/container/src/dndExec.c → source/dnode/mgmt/main/src/dndExec.c

@@ -16,35 +16,30 @@
 #define _DEFAULT_SOURCE
 #include "dndInt.h"
 
-static void dndResetLog(SMgmtWrapper *pMgmt) {
-  char logname[24] = {0};
-  snprintf(logname, sizeof(logname), "%slog", pMgmt->name);
-
-  dInfo("node:%s, reset log to %s in child process", pMgmt->name, logname);
-  taosCloseLog();
-  taosInitLog(logname, 1);
-}
-
 static bool dndRequireNode(SMgmtWrapper *pWrapper) {
-  bool required = false;
-  int32_t code =(*pWrapper->fp.requiredFp)(pWrapper, &required);
+  bool    required = false;
+  int32_t code = (*pWrapper->fp.requiredFp)(pWrapper, &required);
   if (!required) {
-    dDebug("node:%s, no need to start", pWrapper->name);
+    dDebug("node:%s, does not require startup", pWrapper->name);
   } else {
-    dDebug("node:%s, need to start", pWrapper->name);
+    dDebug("node:%s, needs to be started", pWrapper->name);
   }
   return required;
 }
 
 int32_t dndOpenNode(SMgmtWrapper *pWrapper) {
-  int32_t code = (*pWrapper->fp.openFp)(pWrapper);
-  if (code != 0) {
+  if (taosMkDir(pWrapper->path) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("node:%s, failed to create dir:%s since %s", pWrapper->name, pWrapper->path, terrstr());
+    return -1;
+  }
+
+  if ((*pWrapper->fp.openFp)(pWrapper) != 0) {
     dError("node:%s, failed to open since %s", pWrapper->name, terrstr());
     return -1;
-  } else {
-    dDebug("node:%s, has been opened", pWrapper->name);
   }
 
+  dDebug("node:%s, has been opened", pWrapper->name);
   pWrapper->deployed = true;
   return 0;
 }
@@ -70,54 +65,6 @@ void dndCloseNode(SMgmtWrapper *pWrapper) {
   dDebug("node:%s, has been closed", pWrapper->name);
 }
 
-static int32_t dndRunInSingleProcess(SDnode *pDnode) {
-  dInfo("dnode run in single process mode");
-
-  for (ENodeType n = 0; n < NODE_MAX; ++n) {
-    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
-    pWrapper->required = dndRequireNode(pWrapper);
-    if (!pWrapper->required) continue;
-    SMsgCb msgCb = dndCreateMsgcb(pWrapper);
-    tmsgSetDefaultMsgCb(&msgCb);
-
-    if (taosMkDir(pWrapper->path) != 0) {
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      dError("failed to create dir:%s since %s", pWrapper->path, terrstr());
-      return -1;
-    }
-
-    dInfo("node:%s, will start in single process", pWrapper->name);
-    pWrapper->procType = PROC_SINGLE;
-    if (dndOpenNode(pWrapper) != 0) {
-      dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
-      return -1;
-    }
-  }
-
-  dndSetStatus(pDnode, DND_STAT_RUNNING);
-
-  for (ENodeType n = 0; n < NODE_MAX; ++n) {
-    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
-    if (!pWrapper->required) continue;
-    if (pWrapper->fp.startFp == NULL) continue;
-    if ((*pWrapper->fp.startFp)(pWrapper) != 0) {
-      dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
-      return -1;
-    }
-  }
-
-  return 0;
-}
-
-static void dndClearNodesExecpt(SDnode *pDnode, ENodeType except) {
-  // dndCleanupServer(pDnode);
-  for (ENodeType n = 0; n < NODE_MAX; ++n) {
-    if (except == n) continue;
-    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
-    pWrapper->required = false;
-  }
-}
-
 static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen,
                                  ProcFuncType ftype) {
   SRpcMsg *pRpc = &pMsg->rpcMsg;
@@ -163,116 +110,234 @@ static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t
   taosMemoryFree(pMsg);
 }
 
-static int32_t dndRunInMultiProcess(SDnode *pDnode) {
-  dInfo("dnode run in multi process mode");
+static int32_t dndNewProc(SMgmtWrapper *pWrapper, ENodeType n) {
+  char  tstr[8] = {0};
+  char *args[6] = {0};
+  snprintf(tstr, sizeof(tstr), "%d", n);
+  args[1] = "-c";
+  args[2] = configDir;
+  args[3] = "-n";
+  args[4] = tstr;
+  args[5] = NULL;
+
+  int32_t pid = taosNewProc(args);
+  if (pid <= 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("node:%s, failed to exec in new process since %s", pWrapper->name, terrstr());
+    return -1;
+  }
 
-  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+  pWrapper->procId = pid;
+  dInfo("node:%s, continue running in new process:%d", pWrapper->name, pid);
+  return 0;
+}
+
+static SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper) {
+  SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)dndConsumeChildQueue,
+                  .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem,
+                  .childFreeHeadFp = (ProcFreeFp)taosFreeQitem,
+                  .childMallocBodyFp = (ProcMallocFp)rpcMallocCont,
+                  .childFreeBodyFp = (ProcFreeFp)rpcFreeCont,
+                  .parentConsumeFp = (ProcConsumeFp)dndConsumeParentQueue,
+                  .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc,
+                  .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree,
+                  .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont,
+                  .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont,
+                  .shm = pWrapper->shm,
+                  .pParent = pWrapper,
+                  .name = pWrapper->name};
+  return cfg;
+}
+
+static int32_t dndRunInSingleProcess(SDnode *pDnode) {
+  dInfo("dnode run in single process");
+
+  for (ENodeType n = DNODE; n < NODE_MAX; ++n) {
     SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
     pWrapper->required = dndRequireNode(pWrapper);
     if (!pWrapper->required) continue;
 
-    SMsgCb msgCb = dndCreateMsgcb(pWrapper);
-    tmsgSetDefaultMsgCb(&msgCb);
+    if (dndOpenNode(pWrapper) != 0) {
+      dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
+      return -1;
+    }
+  }
+
+  dndSetStatus(pDnode, DND_STAT_RUNNING);
+
+  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+    if (!pWrapper->required) continue;
+    if (pWrapper->fp.startFp == NULL) continue;
+    if ((*pWrapper->fp.startFp)(pWrapper) != 0) {
+      dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
+      return -1;
+    }
+  }
+
+  dInfo("TDengine initialized successfully");
+  dndReportStartup(pDnode, "TDengine", "initialized successfully");
+  while (1) {
+    if (pDnode->event == DND_EVENT_STOP) {
+      dInfo("dnode is about to stop");
+      break;
+    }
+    taosMsleep(100);
+  }
+
+  return 0;
+}
 
-    if (taosMkDir(pWrapper->path) != 0) {
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      dError("failed to create dir:%s since %s", pWrapper->path, terrstr());
+static int32_t dndRunInParentProcess(SDnode *pDnode) {
+  dInfo("dnode run in parent process");
+  SMgmtWrapper *pDWrapper = &pDnode->wrappers[DNODE];
+  if (dndOpenNode(pDWrapper) != 0) {
+    dError("node:%s, failed to start since %s", pDWrapper->name, terrstr());
+    return -1;
+  }
+
+  for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+    pWrapper->required = dndRequireNode(pWrapper);
+    if (!pWrapper->required) continue;
+
+    int32_t shmsize = 1024 * 1024 * 2;  // size will be a configuration item
+    if (taosCreateShm(&pWrapper->shm, shmsize) != 0) {
+      terrno = TAOS_SYSTEM_ERROR(terrno);
+      dError("node:%s, failed to create shm size:%d since %s", pWrapper->name, shmsize, terrstr());
+      return -1;
+    }
+    dInfo("node:%s, shm:%d is created, size:%d", pWrapper->name, pWrapper->shm.id, shmsize);
+
+    SProcCfg cfg = dndGenProcCfg(pWrapper);
+    cfg.isChild = false;
+    pWrapper->procType = PROC_PARENT;
+    pWrapper->pProc = taosProcInit(&cfg);
+    if (pWrapper->pProc == NULL) {
+      dError("node:%s, failed to create proc since %s", pWrapper->name, terrstr());
       return -1;
     }
+  }
 
-    if (n == DNODE) {
-      dInfo("node:%s, will start in parent process", pWrapper->name);
-      pWrapper->procType = PROC_SINGLE;
-      if (dndOpenNode(pWrapper) != 0) {
-        dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
+  if (dndWriteShmFile(pDnode) != 0) {
+    dError("failed to write runtime file since %s", terrstr());
+    return -1;
+  }
+
+  for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+    if (!pWrapper->required) continue;
+
+    if (pDnode->ntype == NODE_MAX) {
+      dInfo("node:%s, should be started manually", pWrapper->name);
+    } else {
+      if (dndNewProc(pWrapper, n) != 0) {
         return -1;
       }
-      continue;
     }
 
-    SProcCfg  cfg = {.childQueueSize = 1024 * 1024 * 2,  // size will be a configuration item
-                     .childConsumeFp = (ProcConsumeFp)dndConsumeChildQueue,
-                     .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem,
-                     .childFreeHeadFp = (ProcFreeFp)taosFreeQitem,
-                     .childMallocBodyFp = (ProcMallocFp)rpcMallocCont,
-                     .childFreeBodyFp = (ProcFreeFp)rpcFreeCont,
-                     .parentQueueSize = 1024 * 1024 * 2,  // size will be a configuration item
-                     .parentConsumeFp = (ProcConsumeFp)dndConsumeParentQueue,
-                     .parentdMallocHeadFp = (ProcMallocFp)taosMemoryMalloc,
-                     .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree,
-                     .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont,
-                     .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont,
-                     .pParent = pWrapper,
-                     .name = pWrapper->name};
-    SProcObj *pProc = taosProcInit(&cfg);
-    if (pProc == NULL) {
-      dError("node:%s, failed to fork since %s", pWrapper->name, terrstr());
+    if (taosProcRun(pWrapper->pProc) != 0) {
+      dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr());
       return -1;
     }
+  }
 
-    pWrapper->pProc = pProc;
+  dndSetStatus(pDnode, DND_STAT_RUNNING);
 
-    if (taosProcIsChild(pProc)) {
-      dInfo("node:%s, will start in child process", pWrapper->name);
-      pWrapper->procType = PROC_CHILD;
-      dndResetLog(pWrapper);
+  if ((*pDWrapper->fp.startFp)(pDWrapper) != 0) {
+    dError("node:%s, failed to start since %s", pDWrapper->name, terrstr());
+    return -1;
+  }
 
-      dInfo("node:%s, clean up resources inherited from parent", pWrapper->name);
-      dndClearNodesExecpt(pDnode, n);
+  dInfo("TDengine initialized successfully");
+  dndReportStartup(pDnode, "TDengine", "initialized successfully");
 
-      dInfo("node:%s, will be initialized in child process", pWrapper->name);
-      if (dndOpenNode(pWrapper) != 0) {
-        dInfo("node:%s, failed to init in child process since %s", pWrapper->name, terrstr());
-        return -1;
+  while (1) {
+    if (pDnode->event == DND_EVENT_STOP) {
+      dInfo("dnode is about to stop");
+      for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) {
+        SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+        if (!pWrapper->required) continue;
+        if (pDnode->ntype == NODE_MAX) continue;
+
+        if (pWrapper->procId > 0 && taosProcExist(pWrapper->procId)) {
+          dInfo("node:%s, send kill signal to the child process:%d", pWrapper->name, pWrapper->procId);
+          taosKillProc(pWrapper->procId);
+        }
       }
 
-      if (taosProcRun(pProc) != 0) {
-        dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr());
-        return -1;
+      for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) {
+        SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+        if (!pWrapper->required) continue;
+        if (pDnode->ntype == NODE_MAX) continue;
+
+        if (pWrapper->procId > 0 && taosProcExist(pWrapper->procId)) {
+          dInfo("node:%s, wait for child process:%d to stop", pWrapper->name, pWrapper->procId);
+          taosWaitProc(pWrapper->procId);
+          dInfo("node:%s, child process:%d is stopped", pWrapper->name, pWrapper->procId);
+        }
       }
       break;
     } else {
-      dInfo("node:%s, will not start in parent process, child pid:%d", pWrapper->name, taosProcChildId(pProc));
-      pWrapper->procType = PROC_PARENT;
-      if (taosProcRun(pProc) != 0) {
-        dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr());
-        return -1;
+      for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) {
+        SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+        if (!pWrapper->required) continue;
+        if (pDnode->ntype == NODE_MAX) continue;
+
+        if (pWrapper->procId <= 0 || !taosProcExist(pWrapper->procId)) {
+          dInfo("node:%s, process:%d is killed and needs to be restarted", pWrapper->name, pWrapper->procId);
+          dndNewProc(pWrapper, n);
+        }
       }
     }
+
+    taosMsleep(100);
   }
 
-  dndSetStatus(pDnode, DND_STAT_RUNNING);
+  return 0;
+}
 
-  for (ENodeType n = 0; n < NODE_MAX; ++n) {
-    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
-    if (!pWrapper->required) continue;
-    if (pWrapper->fp.startFp == NULL) continue;
-    if (pWrapper->procType == PROC_PARENT && n != DNODE) continue;
-    if (pWrapper->procType == PROC_CHILD && n == DNODE) continue;
+static int32_t dndRunInChildProcess(SDnode *pDnode) {
+  SMgmtWrapper *pWrapper = &pDnode->wrappers[pDnode->ntype];
+  dInfo("%s run in child process", pWrapper->name);
+
+  pWrapper->required = dndRequireNode(pWrapper);
+  if (!pWrapper->required) {
+    dError("%s does not require startup", pWrapper->name);
+    return -1;
+  }
+
+  SMsgCb msgCb = dndCreateMsgcb(pWrapper);
+  tmsgSetDefaultMsgCb(&msgCb);
+  pWrapper->procType = PROC_CHILD;
+
+  if (dndOpenNode(pWrapper) != 0) {
+    dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
+    return -1;
+  }
+
+  SProcCfg cfg = dndGenProcCfg(pWrapper);
+  cfg.isChild = true;
+  pWrapper->pProc = taosProcInit(&cfg);
+  if (pWrapper->pProc == NULL) {
+    dError("node:%s, failed to create proc since %s", pWrapper->name, terrstr());
+    return -1;
+  }
+
+  if (pWrapper->fp.startFp != NULL) {
     if ((*pWrapper->fp.startFp)(pWrapper) != 0) {
       dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
       return -1;
     }
   }
 
-  return 0;
-}
-
-int32_t dndRun(SDnode *pDnode) {
-  if (tsMultiProcess == 0) {
-    if (dndRunInSingleProcess(pDnode) != 0) {
-      dError("failed to run dnode in single process mode since %s", terrstr());
-      return -1;
-    }
-  } else {
-    if (dndRunInMultiProcess(pDnode) != 0) {
-      dError("failed to run dnode in multi process mode since %s", terrstr());
-      return -1;
-    }
+  if (taosProcRun(pWrapper->pProc) != 0) {
+    dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr());
+    return -1;
   }
 
+  dInfo("TDengine initialized successfully");
   dndReportStartup(pDnode, "TDengine", "initialized successfully");
-
   while (1) {
     if (pDnode->event == DND_EVENT_STOP) {
       dInfo("dnode is about to stop");
@@ -280,6 +345,16 @@ int32_t dndRun(SDnode *pDnode) {
     }
     taosMsleep(100);
   }
+}
+
+int32_t dndRun(SDnode *pDnode) {
+  if (!tsMultiProcess) {
+    return dndRunInSingleProcess(pDnode);
+  } else if (pDnode->ntype == DNODE || pDnode->ntype == NODE_MAX) {
+    return dndRunInParentProcess(pDnode);
+  } else {
+    return dndRunInChildProcess(pDnode);
+  }
 
   return 0;
 }

source/dnode/mgmt/main/src/dndFile.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http:www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndInt.h"
+
+#define MAXLEN 1024
+
+int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed) {
+  int32_t   code = TSDB_CODE_NODE_PARSE_FILE_ERROR;
+  int64_t   len = 0;
+  char      content[MAXLEN + 1] = {0};
+  cJSON    *root = NULL;
+  char      file[PATH_MAX];
+  TdFilePtr pFile = NULL;
+
+  snprintf(file, sizeof(file), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name);
+  pFile = taosOpenFile(file, TD_FILE_READ);
+  if (pFile == NULL) {
+    dDebug("file %s not exist", file);
+    code = 0;
+    goto _OVER;
+  }
+
+  len = taosReadFile(pFile, content, MAXLEN);
+  if (len <= 0) {
+    dError("failed to read %s since content is null", file);
+    goto _OVER;
+  }
+
+  root = cJSON_Parse(content);
+  if (root == NULL) {
+    dError("failed to read %s since invalid json format", file);
+    goto _OVER;
+  }
+
+  cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
+  if (!deployed || deployed->type != cJSON_Number) {
+    dError("failed to read %s since deployed not found", file);
+    goto _OVER;
+  }
+  *pDeployed = deployed->valueint != 0;
+
+  dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed);
+  code = 0;
+
+_OVER:
+  if (root != NULL) cJSON_Delete(root);
+  if (pFile != NULL) taosCloseFile(&pFile);
+
+  terrno = code;
+  return code;
+}
+
+int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed) {
+  int32_t   code = -1;
+  int32_t   len = 0;
+  char      content[MAXLEN + 1] = {0};
+  char      file[PATH_MAX] = {0};
+  char      realfile[PATH_MAX] = {0};
+  TdFilePtr pFile = NULL;
+
+  snprintf(file, sizeof(file), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name);
+  snprintf(realfile, sizeof(realfile), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name);
+
+  pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to write %s since %s", file, terrstr());
+    goto _OVER;
+  }
+
+  len += snprintf(content + len, MAXLEN - len, "{\n");
+  len += snprintf(content + len, MAXLEN - len, "  \"deployed\": %d\n", deployed);
+  len += snprintf(content + len, MAXLEN - len, "}\n");
+
+  if (taosWriteFile(pFile, content, len) != len) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to write file:%s since %s", file, terrstr());
+    goto _OVER;
+  }
+
+  if (taosFsyncFile(pFile) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to fsync file:%s since %s", file, terrstr());
+    goto _OVER;
+  }
+
+  taosCloseFile(&pFile);
+
+  if (taosRenameFile(file, realfile) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to rename %s since %s", file, terrstr());
+    return -1;
+  }
+
+  dInfo("successed to write %s, deployed:%d", realfile, deployed);
+  code = 0;
+
+_OVER:
+  if (pFile != NULL) {
+    taosCloseFile(&pFile);
+  }
+
+  return code;
+}
+
+TdFilePtr dndCheckRunning(const char *dataDir) {
+  char filepath[PATH_MAX] = {0};
+  snprintf(filepath, sizeof(filepath), "%s%s.running", dataDir, TD_DIRSEP);
+
+  TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to lock file:%s since %s", filepath, terrstr());
+    return NULL;
+  }
+
+  int32_t ret = taosLockFile(pFile);
+  if (ret != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to lock file:%s since %s", filepath, terrstr());
+    taosCloseFile(&pFile);
+    return NULL;
+  }
+
+  dDebug("file:%s is locked", filepath);
+  return pFile;
+}
+
+int32_t dndReadShmFile(SDnode *pDnode) {
+  int32_t   code = -1;
+  char      itemName[24] = {0};
+  char      content[MAXLEN + 1] = {0};
+  char      file[PATH_MAX] = {0};
+  cJSON    *root = NULL;
+  TdFilePtr pFile = NULL;
+
+  snprintf(file, sizeof(file), "%s%s.shmfile", pDnode->dataDir, TD_DIRSEP);
+  pFile = taosOpenFile(file, TD_FILE_READ);
+  if (pFile == NULL) {
+    dDebug("file %s not exist", file);
+    code = 0;
+    goto _OVER;
+  }
+
+  if (taosReadFile(pFile, content, MAXLEN) > 0) {
+    root = cJSON_Parse(content);
+    if (root == NULL) {
+      terrno = TSDB_CODE_NODE_PARSE_FILE_ERROR;
+      dError("failed to read %s since invalid json format", file);
+      goto _OVER;
+    }
+
+    for (ENodeType ntype = DNODE + 1; ntype < NODE_MAX; ++ntype) {
+      snprintf(itemName, sizeof(itemName), "%s_shmid", dndNodeProcStr(ntype));
+      cJSON *shmid = cJSON_GetObjectItem(root, itemName);
+      if (shmid && shmid->type == cJSON_Number) {
+        pDnode->wrappers[ntype].shm.id = shmid->valueint;
+      }
+
+      snprintf(itemName, sizeof(itemName), "%s_shmsize", dndNodeProcStr(ntype));
+      cJSON *shmsize = cJSON_GetObjectItem(root, itemName);
+      if (shmsize && shmsize->type == cJSON_Number) {
+        pDnode->wrappers[ntype].shm.size = shmsize->valueint;
+      }
+    }
+  }
+
+  if (!tsMultiProcess || pDnode->ntype == DNODE || pDnode->ntype == NODE_MAX) {
+    for (ENodeType ntype = DNODE; ntype < NODE_MAX; ++ntype) {
+      SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
+      if (pWrapper->shm.id >= 0) {
+        dDebug("shmid:%d, is closed, size:%d", pWrapper->shm.id, pWrapper->shm.size);
+        taosDropShm(&pWrapper->shm);
+      }
+    }
+  } else {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[pDnode->ntype];
+    if (taosAttachShm(&pWrapper->shm) != 0) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      dError("shmid:%d, failed to attach shm since %s", pWrapper->shm.id, terrstr());
+      goto _OVER;
+    }
+    dInfo("node:%s, shmid:%d is attached, size:%d", pWrapper->name, pWrapper->shm.id, pWrapper->shm.size);
+  }
+
+  dDebug("successed to load %s", file);
+  code = 0;
+
+_OVER:
+  if (root != NULL) cJSON_Delete(root);
+  if (pFile != NULL) taosCloseFile(&pFile);
+
+  return code;
+}
+
+int32_t dndWriteShmFile(SDnode *pDnode) {
+  int32_t   code = -1;
+  int32_t   len = 0;
+  char      content[MAXLEN + 1] = {0};
+  char      file[PATH_MAX] = {0};
+  char      realfile[PATH_MAX] = {0};
+  TdFilePtr pFile = NULL;
+
+  snprintf(file, sizeof(file), "%s%s.shmfile.bak", pDnode->dataDir, TD_DIRSEP);
+  snprintf(realfile, sizeof(realfile), "%s%s.shmfile", pDnode->dataDir, TD_DIRSEP);
+
+  pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to open file:%s since %s", file, terrstr());
+    goto _OVER;
+  }
+
+  len += snprintf(content + len, MAXLEN - len, "{\n");
+  for (ENodeType ntype = DNODE + 1; ntype < NODE_MAX; ++ntype) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
+    len += snprintf(content + len, MAXLEN - len, "  \"%s_shmid\":%d,\n", dndNodeProcStr(ntype), pWrapper->shm.id);
+    if (ntype == NODE_MAX - 1) {
+      len += snprintf(content + len, MAXLEN - len, "  \"%s_shmsize\":%d\n", dndNodeProcStr(ntype), pWrapper->shm.size);
+    } else {
+      len += snprintf(content + len, MAXLEN - len, "  \"%s_shmsize\":%d,\n", dndNodeProcStr(ntype), pWrapper->shm.size);
+    }
+  }
+  len += snprintf(content + len, MAXLEN - len, "}\n");
+
+  if (taosWriteFile(pFile, content, len) != len) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to write file:%s since %s", file, terrstr());
+    goto _OVER;
+  }
+
+  if (taosFsyncFile(pFile) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to fsync file:%s since %s", file, terrstr());
+    goto _OVER;
+  }
+
+  taosCloseFile(&pFile);
+
+  if (taosRenameFile(file, realfile) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to rename %s to %s since %s", file, realfile, terrstr());
+    return -1;
+  }
+
+  dInfo("successed to write %s", realfile);
+  code = 0;
+
+_OVER:
+  if (pFile != NULL) {
+    taosCloseFile(&pFile);
+  }
+
+  return code;
+}

source/dnode/mgmt/container/src/dndObj.c → source/dnode/mgmt/main/src/dndInt.c

@@ -16,7 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "dndInt.h"
 
-static int32_t dndInitMemory(SDnode *pDnode, const SDnodeOpt *pOption) {
+static int32_t dndInitVars(SDnode *pDnode, const SDnodeOpt *pOption) {
   pDnode->numOfSupportVnodes = pOption->numOfSupportVnodes;
   pDnode->serverPort = pOption->serverPort;
   pDnode->dataDir = strdup(pOption->dataDir);
@@ -24,8 +24,9 @@ static int32_t dndInitMemory(SDnode *pDnode, const SDnodeOpt *pOption) {
   pDnode->localFqdn = strdup(pOption->localFqdn);
   pDnode->firstEp = strdup(pOption->firstEp);
   pDnode->secondEp = strdup(pOption->secondEp);
-  pDnode->pDisks = pOption->pDisks;
+  pDnode->disks = pOption->disks;
   pDnode->numOfDisks = pOption->numOfDisks;
+  pDnode->ntype = pOption->ntype;
   pDnode->rebootTime = taosGetTimestampMs();
 
   if (pDnode->dataDir == NULL || pDnode->localEp == NULL || pDnode->localFqdn == NULL || pDnode->firstEp == NULL ||
@@ -33,18 +34,26 @@ static int32_t dndInitMemory(SDnode *pDnode, const SDnodeOpt *pOption) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
+
+  if (!tsMultiProcess || pDnode->ntype == DNODE || pDnode->ntype == NODE_MAX) {
+    pDnode->lockfile = dndCheckRunning(pDnode->dataDir);
+    if (pDnode->lockfile == NULL) {
+      return -1;
+    }
+  }
+
   return 0;
 }
 
-static void dndClearMemory(SDnode *pDnode) {
+static void dndClearVars(SDnode *pDnode) {
   for (ENodeType n = 0; n < NODE_MAX; ++n) {
     SMgmtWrapper *pMgmt = &pDnode->wrappers[n];
     taosMemoryFreeClear(pMgmt->path);
   }
-  if (pDnode->pLockFile != NULL) {
-    taosUnLockFile(pDnode->pLockFile);
-    taosCloseFile(&pDnode->pLockFile);
-    pDnode->pLockFile = NULL;
+  if (pDnode->lockfile != NULL) {
+    taosUnLockFile(pDnode->lockfile);
+    taosCloseFile(&pDnode->lockfile);
+    pDnode->lockfile = NULL;
   }
   taosMemoryFreeClear(pDnode->localEp);
   taosMemoryFreeClear(pDnode->localFqdn);
@@ -52,11 +61,11 @@ static void dndClearMemory(SDnode *pDnode) {
   taosMemoryFreeClear(pDnode->secondEp);
   taosMemoryFreeClear(pDnode->dataDir);
   taosMemoryFree(pDnode);
-  dDebug("dnode object memory is cleared, data:%p", pDnode);
+  dDebug("dnode memory is cleared, data:%p", pDnode);
 }
 
 SDnode *dndCreate(const SDnodeOpt *pOption) {
-  dInfo("start to create dnode object");
+  dDebug("start to create dnode object");
   int32_t code = -1;
   char    path[PATH_MAX] = {0};
   SDnode *pDnode = NULL;
@@ -67,26 +76,12 @@ SDnode *dndCreate(const SDnodeOpt *pOption) {
     goto _OVER;
   }
 
-  if (dndInitMemory(pDnode, pOption) != 0) {
+  if (dndInitVars(pDnode, pOption) != 0) {
+    dError("failed to init variables since %s", terrstr());
     goto _OVER;
   }
 
   dndSetStatus(pDnode, DND_STAT_INIT);
-  pDnode->pLockFile = dndCheckRunning(pDnode->dataDir);
-  if (pDnode->pLockFile == NULL) {
-    goto _OVER;
-  }
-
-  if (dndInitServer(pDnode) != 0) {
-    dError("failed to init trans server since %s", terrstr());
-    goto _OVER;
-  }
-
-  if (dndInitClient(pDnode) != 0) {
-    dError("failed to init trans client since %s", terrstr());
-    goto _OVER;
-  }
-
   dmGetMgmtFp(&pDnode->wrappers[DNODE]);
   mmGetMgmtFp(&pDnode->wrappers[MNODE]);
   vmGetMgmtFp(&pDnode->wrappers[VNODES]);
@@ -94,14 +89,11 @@ SDnode *dndCreate(const SDnodeOpt *pOption) {
   smGetMgmtFp(&pDnode->wrappers[SNODE]);
   bmGetMgmtFp(&pDnode->wrappers[BNODE]);
 
-  if (dndInitMsgHandle(pDnode) != 0) {
-    goto _OVER;
-  }
-
   for (ENodeType n = 0; n < NODE_MAX; ++n) {
     SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
     snprintf(path, sizeof(path), "%s%s%s", pDnode->dataDir, TD_DIRSEP, pWrapper->name);
     pWrapper->path = strdup(path);
+    pWrapper->shm.id = -1;
     pWrapper->pDnode = pDnode;
     if (pWrapper->path == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -112,15 +104,27 @@ SDnode *dndCreate(const SDnodeOpt *pOption) {
     taosInitRWLatch(&pWrapper->latch);
   }
 
+  if (dndInitMsgHandle(pDnode) != 0) {
+    dError("failed to msg handles since %s", terrstr());
+    goto _OVER;
+  }
+
+  if (dndReadShmFile(pDnode) != 0) {
+    dError("failed to read shm file since %s", terrstr());
+    goto _OVER;
+  }
+
+  SMsgCb msgCb = dndCreateMsgcb(&pDnode->wrappers[0]);
+  tmsgSetDefaultMsgCb(&msgCb);
+
+  dInfo("dnode is created, data:%p", pDnode);
   code = 0;
 
 _OVER:
   if (code != 0 && pDnode) {
-    dndClearMemory(pDnode);
+    dndClearVars(pDnode);
     pDnode = NULL;
-    dError("failed to create dnode object since %s", terrstr());
-  } else {
-    dInfo("dnode object is created, data:%p", pDnode);
+    dError("failed to create dnode since %s", terrstr());
   }
 
   return pDnode;
@@ -137,21 +141,20 @@ void dndClose(SDnode *pDnode) {
   dInfo("start to close dnode, data:%p", pDnode);
   dndSetStatus(pDnode, DND_STAT_STOPPED);
 
-  dndCleanupServer(pDnode);
-  dndCleanupClient(pDnode);
-
   for (ENodeType n = 0; n < NODE_MAX; ++n) {
     SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
     dndCloseNode(pWrapper);
   }
 
-  dndClearMemory(pDnode);
-  dInfo("dnode object is closed, data:%p", pDnode);
+  dndClearVars(pDnode);
+  dInfo("dnode is closed, data:%p", pDnode);
 }
 
 void dndHandleEvent(SDnode *pDnode, EDndEvent event) {
-  dInfo("dnode object receive event %d, data:%p", event, pDnode);
-  pDnode->event = event;
+  dInfo("dnode receive %s event, data:%p", dndEventStr(event), pDnode);
+  if (event == DND_EVENT_STOP) {
+    pDnode->event = event;
+  }
 }
 
 SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType ntype) {

source/dnode/mgmt/container/src/dndMsg.c → source/dnode/mgmt/main/src/dndMsg.c

@@ -66,7 +66,7 @@ void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) {
     dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user);
     code = (*msgFp)(pWrapper, pMsg);
   } else if (pWrapper->procType == PROC_PARENT) {
-    dTrace("msg:%p, is created and will put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle,
+    dTrace("msg:%p, is created and put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle,
            pRpc->ahandle, pMsg->user);
     code = taosProcPutToChildQ(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, PROC_REQ);
   } else {

source/dnode/mgmt/main/src/dndUtil.c → source/dnode/mgmt/main/src/dndStr.c

@@ -14,42 +14,64 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "dndMain.h"
+#include "dndInt.h"
 
-void dndGenerateGrant() {
-#if 0
-  grantParseParameter();
-#endif
+const char *dndStatStr(EDndStatus status) {
+  switch (status) {
+    case DND_STAT_INIT:
+      return "init";
+    case DND_STAT_RUNNING:
+      return "running";
+    case DND_STAT_STOPPED:
+      return "stopped";
+    default:
+      return "unknown";
+  }
 }
 
-void dndPrintVersion() {
-#ifdef TD_ENTERPRISE
-  char *releaseName = "enterprise";
-#else
-  char *releaseName = "community";
-#endif
-  printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version);
-  printf("gitinfo: %s\n", gitinfo);
-  printf("builuInfo: %s\n", buildinfo);
+const char *dndNodeLogStr(ENodeType ntype) {
+  switch (ntype) {
+    case VNODES:
+      return "vnode";
+    case QNODE:
+      return "qnode";
+    case SNODE:
+      return "snode";
+    case MNODE:
+      return "mnode";
+    case BNODE:
+      return "bnode";
+    default:
+      return "taosd";
+  }
 }
 
-void dndDumpCfg() {
-  SConfig *pCfg = taosGetCfg();
-  cfgDumpCfg(pCfg, 0, 1);
+const char *dndNodeProcStr(ENodeType ntype) {
+  switch (ntype) {
+    case VNODES:
+      return "taosv";
+    case QNODE:
+      return "taosq";
+    case SNODE:
+      return "taoss";
+    case MNODE:
+      return "taosm";
+    case BNODE:
+      return "taosb";
+    default:
+      return "taosd";
+  }
 }
 
-SDnodeOpt dndGetOpt() {
-  SConfig  *pCfg = taosGetCfg();
-  SDnodeOpt option = {0};
-
-  option.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32;
-  tstrncpy(option.dataDir, tsDataDir, sizeof(option.dataDir));
-  tstrncpy(option.firstEp, tsFirst, sizeof(option.firstEp));
-  tstrncpy(option.secondEp, tsSecond, sizeof(option.firstEp));
-  option.serverPort = tsServerPort;
-  tstrncpy(option.localFqdn, tsLocalFqdn, sizeof(option.localFqdn));
-  snprintf(option.localEp, sizeof(option.localEp), "%s:%u", option.localFqdn, option.serverPort);
-  option.pDisks = tsDiskCfg;
-  option.numOfDisks = tsDiskCfgNum;
+const char *dndEventStr(EDndEvent ev) {
+  switch (ev) {
+    case DND_EVENT_START:
+      return "start";
+    case DND_EVENT_STOP:
+      return "stop";
+    case DND_EVENT_CHILD:
+      return "child";
+    default:
+      return "UNKNOWN";
+  }
+}
\ No newline at end of file
-  return option;
-}

source/dnode/mgmt/test/vnode/vnode.cpp

@@ -164,37 +164,37 @@ TEST_F(DndTestVnode, 03_Create_Stb) {
     req.keep = 0;
     req.type = TD_SUPER_TABLE;
 
-    SSchema schemas[5] = {0};
+    SSchemaEx schemas[2] = {0};
     {
-      SSchema* pSchema = &schemas[0];
+      SSchemaEx* pSchema = &schemas[0];
       pSchema->bytes = htonl(8);
       pSchema->type = TSDB_DATA_TYPE_TIMESTAMP;
       strcpy(pSchema->name, "ts");
     }
 
     {
-      SSchema* pSchema = &schemas[1];
+      SSchemaEx* pSchema = &schemas[1];
       pSchema->bytes = htonl(4);
       pSchema->type = TSDB_DATA_TYPE_INT;
       strcpy(pSchema->name, "col1");
     }
-
+    SSchema tagSchemas[3] = {0};
     {
-      SSchema* pSchema = &schemas[2];
+      SSchema* pSchema = &tagSchemas[0];
       pSchema->bytes = htonl(2);
       pSchema->type = TSDB_DATA_TYPE_TINYINT;
       strcpy(pSchema->name, "tag1");
     }
 
     {
-      SSchema* pSchema = &schemas[3];
+      SSchema* pSchema = &tagSchemas[1];
       pSchema->bytes = htonl(8);
       pSchema->type = TSDB_DATA_TYPE_BIGINT;
       strcpy(pSchema->name, "tag2");
     }
 
     {
-      SSchema* pSchema = &schemas[4];
+      SSchema* pSchema = &tagSchemas[2];
       pSchema->bytes = htonl(16);
       pSchema->type = TSDB_DATA_TYPE_BINARY;
       strcpy(pSchema->name, "tag3");
@@ -204,7 +204,7 @@ TEST_F(DndTestVnode, 03_Create_Stb) {
     req.stbCfg.nCols = 2;
     req.stbCfg.pSchema = &schemas[0];
     req.stbCfg.nTagCols = 3;
-    req.stbCfg.pTagSchema = &schemas[2];
+    req.stbCfg.pTagSchema = &tagSchemas[0];
 
     int32_t   contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead);
     SMsgHead* pHead = (SMsgHead*)rpcMallocCont(contLen);
@@ -236,37 +236,37 @@ TEST_F(DndTestVnode, 04_Alter_Stb) {
     req.keep = 0;
     req.type = TD_SUPER_TABLE;
 
-    SSchema schemas[5] = {0};
+    SSchemaEx schemas[2] = {0};
     {
-      SSchema* pSchema = &schemas[0];
+      SSchemaEx* pSchema = &schemas[0];
       pSchema->bytes = htonl(8);
       pSchema->type = TSDB_DATA_TYPE_TIMESTAMP;
       strcpy(pSchema->name, "ts");
     }
 
     {
-      SSchema* pSchema = &schemas[1];
+      SSchemaEx* pSchema = &schemas[1];
       pSchema->bytes = htonl(4);
       pSchema->type = TSDB_DATA_TYPE_INT;
       strcpy(pSchema->name, "col1");
     }
-
+    SSchema tagSchemas[3] = {0};
     {
-      SSchema* pSchema = &schemas[2];
+      SSchema* pSchema = &tagSchemas[0];
       pSchema->bytes = htonl(2);
       pSchema->type = TSDB_DATA_TYPE_TINYINT;
       strcpy(pSchema->name, "_tag1");
     }
 
     {
-      SSchema* pSchema = &schemas[3];
+      SSchema* pSchema = &tagSchemas[1];
       pSchema->bytes = htonl(8);
       pSchema->type = TSDB_DATA_TYPE_BIGINT;
       strcpy(pSchema->name, "_tag2");
     }
 
     {
-      SSchema* pSchema = &schemas[4];
+      SSchema* pSchema = &tagSchemas[2];
       pSchema->bytes = htonl(16);
       pSchema->type = TSDB_DATA_TYPE_BINARY;
       strcpy(pSchema->name, "_tag3");
@@ -276,7 +276,7 @@ TEST_F(DndTestVnode, 04_Alter_Stb) {
     req.stbCfg.nCols = 2;
     req.stbCfg.pSchema = &schemas[0];
     req.stbCfg.nTagCols = 3;
-    req.stbCfg.pTagSchema = &schemas[2];
+    req.stbCfg.pTagSchema = &tagSchemas[0];
 
     int32_t   contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead);
     SMsgHead* pHead = (SMsgHead*)rpcMallocCont(contLen);

source/dnode/mgmt/vnode/src/vmInt.c → source/dnode/mgmt/vm/src/vmInt.c

@@ -285,7 +285,7 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) {
   tstrncpy(dCfg.dir, pDnode->dataDir, TSDB_FILENAME_LEN);
   dCfg.level = 0;
   dCfg.primary = 1;
-  SDiskCfg *pDisks = pDnode->pDisks;
+  SDiskCfg *pDisks = pDnode->disks;
   int32_t   numOfDisks = pDnode->numOfDisks;
   if (numOfDisks <= 0 || pDisks == NULL) {
     pDisks = &dCfg;

source/dnode/mgmt/vnode/src/vmWorker.c → source/dnode/mgmt/vm/src/vmWorker.c

@@ -76,7 +76,7 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
   SVnodeObj *pVnode = pInfo->ahandle;
 
   dTrace("msg:%p, will be processed in vnode-fetch queue", pMsg);
-  int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, &pMsg->rpcMsg);
+  int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, &pMsg->rpcMsg, pInfo);
   if (code != 0) {
     vmSendRsp(pVnode->pWrapper, pMsg, code);
     dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
@@ -168,7 +168,7 @@ static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
     taosGetQitem(qall, (void **)&pMsg);
 
     dTrace("msg:%p, will be processed in vnode-merge queue", pMsg);
-    int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, &pMsg->rpcMsg);
+    int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, &pMsg->rpcMsg, pInfo);
     if (code != 0) {
       vmSendRsp(pVnode->pWrapper, pMsg, code);
       dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
@@ -187,7 +187,7 @@ static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueTyp
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
   if (pVnode == NULL) {
     dError("vgId:%d, failed to write msg:%p to vnode-queue since %s", pHead->vgId, pMsg, terrstr());
-    return -1;
+    return terrno;
   }
 
   int32_t code = 0;
@@ -414,8 +414,7 @@ int32_t vmStartWorker(SVnodesMgmt *pMgmt) {
   pWPool->max = maxMergeThreads;
   if (tWWorkerInit(pWPool) != 0) return -1;
 
-  SSingleWorkerCfg cfg = {
-      .min = 1, .max = 1, .name = "vnode-mgmt", .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt};
+  SSingleWorkerCfg cfg = {.min = 1, .max = 1, .name = "vnode-mgmt", .fp = (FItem)vmProcessMgmtQueue, .param = pMgmt};
   if (tSingleWorkerInit(&pMgmt->mgmtWorker, &cfg) != 0) {
     dError("failed to start vnode-mgmt worker since %s", terrstr());
     return -1;

source/dnode/mnode/impl/inc/mndDef.h

@@ -620,13 +620,13 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub)
 
 static FORCE_INLINE void tDeleteSMqSubscribeObj(SMqSubscribeObj* pSub) {
   if (pSub->consumers) {
-    taosArrayDestroyEx(pSub->consumers, (void (*)(void*))tDeleteSMqSubConsumer);
+    //taosArrayDestroyEx(pSub->consumers, (void (*)(void*))tDeleteSMqSubConsumer);
     // taosArrayDestroy(pSub->consumers);
     pSub->consumers = NULL;
   }
 
   if (pSub->unassignedVg) {
-    taosArrayDestroyEx(pSub->unassignedVg, (void (*)(void*))tDeleteSMqConsumerEp);
+    //taosArrayDestroyEx(pSub->unassignedVg, (void (*)(void*))tDeleteSMqConsumerEp);
     // taosArrayDestroy(pSub->unassignedVg);
     pSub->unassignedVg = NULL;
   }
@@ -735,6 +735,9 @@ typedef struct {
   int8_t         createdBy;      // STREAM_CREATED_BY__USER or SMA
   int32_t        fixedSinkVgId;  // 0 for shuffle
   int64_t        smaId;          // 0 for unused
+  int8_t         trigger;
+  int32_t        triggerParam;
+  int64_t        waterMark;
   char*          sql;
   char*          logicalPlan;
   char*          physicalPlan;

source/dnode/mnode/impl/src/mndConsumer.c

@@ -160,6 +160,7 @@ static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) {
 
 static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) {
   mTrace("consumer:%" PRId64 ", perform update action", pOldConsumer->consumerId);
+  pOldConsumer->epoch++;
 
   // TODO handle update
   /*taosWLockLatch(&pOldConsumer->lock);*/

source/dnode/mnode/impl/src/mndDb.c

@@ -1309,7 +1309,7 @@ static int32_t mndGetDbMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMet
   cols++;
 
   pShow->bytes[cols] = 2;
-  pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "days");
   pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
@@ -1444,7 +1444,7 @@ static void dumpDbInfoToPayload(char *data, SDbObj *pDb, SShowObj *pShow, int32_
   cols++;
 
   pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
-  *(int16_t *)pWrite = pDb->cfg.daysPerFile;
+  *(int32_t *)pWrite = pDb->cfg.daysPerFile;
   cols++;
 
   pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);

source/dnode/mnode/impl/src/mndInfoSchema.c

@@ -53,7 +53,7 @@ static const SInfosTableSchema userDBSchema[] = {
     {.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
     {.name = "replica", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
     {.name = "quorum", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
-    {.name = "days", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
+    {.name = "days", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "keep", .bytes = 24 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "cache", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
@@ -89,7 +89,6 @@ static const SInfosTableSchema userStbsSchema[] = {
     {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
     {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "last_update", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
     {.name = "table_comment", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_INT},
 };