Merge branch '3.0' of https://github.com/taosdata/TDengine into feature/tdb

example/src/tmq.c

@@ -28,7 +28,7 @@ int32_t init_env() {
     return -1;
   }
 
-  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1");
+  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2");
   if (taos_errno(pRes) != 0) {
     printf("error in create db, reason:%s\n", taos_errstr(pRes));
     return -1;
@@ -42,25 +42,33 @@ int32_t init_env() {
   }
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int) tags(a int)");
+  pRes =
+      taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)");
   if (taos_errno(pRes) != 0) {
     printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes));
     return -1;
   }
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "create table if not exists tu1 using st1 tags(1)");
+  pRes = taos_query(pConn, "create table if not exists ct0 using st1 tags(1000)");
   if (taos_errno(pRes) != 0) {
     printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes));
     return -1;
   }
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "create table if not exists tu2 using st1 tags(2)");
+  pRes = taos_query(pConn, "create table if not exists ct1 using st1 tags(2000)");
   if (taos_errno(pRes) != 0) {
     printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes));
     return -1;
   }
+
+  pRes = taos_query(pConn, "create table if not exists ct3 using st1 tags(3000)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create child table tu3, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+
   taos_free_result(pRes);
   return 0;
 }
@@ -82,12 +90,40 @@ int32_t create_topic() {
 
   /*const char* sql = "select * from tu1";*/
   /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));*/
-  pRes = taos_query(pConn, "create topic test_stb_topic_1 as select * from tu1");
+  pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1 from ct1");
   if (taos_errno(pRes) != 0) {
-    printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes));
+    printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes));
     return -1;
   }
   taos_free_result(pRes);
+
+#if 0
+  pRes = taos_query(pConn, "insert into tu1 values(now, 1, 1.0, 'bi1')");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to insert, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+  pRes = taos_query(pConn, "insert into tu1 values(now+1d, 1, 1.0, 'bi1')");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to insert, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+  pRes = taos_query(pConn, "insert into tu2 values(now, 2, 2.0, 'bi2')");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to insert, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+  pRes = taos_query(pConn, "insert into tu2 values(now+1d, 2, 2.0, 'bi2')");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to insert, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+#endif
+
   taos_close(pConn);
   return 0;
 }
@@ -115,7 +151,7 @@ tmq_t* build_consumer() {
 
 tmq_list_t* build_topic_list() {
   tmq_list_t* topic_list = tmq_list_new();
-  tmq_list_append(topic_list, "test_stb_topic_1");
+  tmq_list_append(topic_list, "topic_ctb_column");
   return topic_list;
 }
 
@@ -215,8 +251,8 @@ int main(int argc, char* argv[]) {
   if (argc > 1) {
     printf("env init\n");
     code = init_env();
-  }
     create_topic();
+  }
   tmq_t*      tmq = build_consumer();
   tmq_list_t* topic_list = build_topic_list();
   /*perf_loop(tmq, topic_list);*/

example/src/tstream.c

@@ -20,7 +20,7 @@
 #include "taos.h"
 
 int32_t init_env() {
-  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 7010);
+  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   if (pConn == NULL) {
     return -1;
   }
@@ -65,7 +65,7 @@ int32_t init_env() {
 int32_t create_stream() {
   printf("create stream\n");
   TAOS_RES* pRes;
-  TAOS*     pConn = taos_connect("localhost", "root", "taosdata", NULL, 7010);
+  TAOS*     pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   if (pConn == NULL) {
     return -1;
   }

include/common/tcommon.h

@@ -37,6 +37,14 @@ enum {
   TMQ_MSG_TYPE__EP_RSP,
 };
 
+enum {
+  STREAM_TRIGGER__AT_ONCE = 1,
+  STREAM_TRIGGER__WINDOW_CLOSE,
+  STREAM_TRIGGER__BY_COUNT,
+  STREAM_TRIGGER__BY_BATCH_COUNT,
+  STREAM_TRIGGER__BY_EVENT_TIME,
+};
+
 typedef struct {
   uint32_t  numOfTables;
   SArray*   pGroupList;
@@ -59,7 +67,10 @@ typedef struct SDataBlockInfo {
   int32_t     rowSize;
   int16_t     numOfCols;
   int16_t     hasVarCol;
-  union {int64_t uid; int64_t blockId;};
+  union {
+    int64_t uid;
+    int64_t blockId;
+  };
   int64_t groupId;  // no need to serialize
 } SDataBlockInfo;
 
@@ -93,7 +104,7 @@ void*   tDecodeDataBlock(const void* buf, SSDataBlock* pBlock);
 
 int32_t tEncodeDataBlocks(void** buf, const SArray* blocks);
 void*   tDecodeDataBlocks(const void* buf, SArray** blocks);
-void    colDataDestroy(SColumnInfoData* pColData) ;
+void    colDataDestroy(SColumnInfoData* pColData);
 
 static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) {
   // WARNING: do not use info.numOfCols,
@@ -198,8 +209,8 @@ typedef struct SGroupbyExpr {
 } SGroupbyExpr;
 
 enum {
-  FUNC_PARAM_TYPE_VALUE = 0,
-  FUNC_PARAM_TYPE_COLUMN,
+  FUNC_PARAM_TYPE_VALUE = 0x1,
+  FUNC_PARAM_TYPE_COLUMN= 0x2,
 };
 
 typedef struct SFunctParam {

include/common/tmsg.h

@@ -524,6 +524,7 @@ typedef struct {
   int8_t  walLevel;
   int8_t  quorum;
   int8_t  cacheLastRow;
+  int8_t  replications;
 } SAlterDbReq;
 
 int32_t tSerializeSAlterDbReq(void* buf, int32_t bufLen, SAlterDbReq* pReq);

include/common/trow.h

@@ -98,7 +98,7 @@ typedef void *SRow;
 
 typedef struct {
   TDRowValT valType;
-  void *    val;
+  void     *val;
 } SCellVal;
 
 typedef struct {
@@ -158,8 +158,8 @@ typedef struct {
   int16_t nBitmaps;
   int16_t nBoundBitmaps;
   int32_t offset;
-  void *  pBitmap;
-  void *  pOffset;
+  void   *pBitmap;
+  void   *pOffset;
   int32_t extendedRowSize;
 } SRowBuilder;
 
@@ -275,7 +275,7 @@ static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TD
   }
   int16_t nBytes = colIdx / TD_VTYPE_PARTS;
   int16_t nOffset = colIdx & TD_VTYPE_OPTR;
-  char *  pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes);
+  char   *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes);
   switch (nOffset) {
     case 0:
       *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6);
@@ -313,7 +313,7 @@ static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TD
   }
   int16_t nBytes = colIdx / TD_VTYPE_PARTS;
   int16_t nOffset = colIdx & TD_VTYPE_OPTR;
-  char *  pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes);
+  char   *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes);
   switch (nOffset) {
     case 0:
       *pValType = (((*pDestByte) & 0xC0) >> 6);
@@ -620,7 +620,7 @@ static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowVa
   if (tdValIsNorm(valType, val, colType)) {
     // ts key stored in STSRow.ts
     SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset);
-    char *     ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row));
+    char      *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row));
     pColIdx->colId = colId;
     pColIdx->offset = TD_ROW_LEN(row);  // the offset include the TD_ROW_HEAD_LEN
 
@@ -638,7 +638,7 @@ static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowVa
   // NULL/None value
   else {
     SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset);
-    char *     ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row));
+    char      *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row));
     pColIdx->colId = colId;
     pColIdx->offset = TD_ROW_LEN(row);  // the offset include the TD_ROW_HEAD_LEN
     const void *nullVal = getNullValue(colType);
@@ -775,8 +775,8 @@ static FORCE_INLINE int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, v
 
 typedef struct {
   STSchema *pSchema;
-  STSRow *  pRow;
-  void *    pBitmap;
+  STSRow   *pRow;
+  void     *pBitmap;
   uint32_t  offset;
   col_id_t  maxColId;
   col_id_t  colIdx;  // [PRIMARYKEY_TIMESTAMP_COL_ID, nSchemaCols], PRIMARYKEY_TIMESTAMP_COL_ID equals 1
@@ -881,7 +881,7 @@ static FORCE_INLINE bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colTy
 // internal
 static FORCE_INLINE bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx,
                                               SCellVal *pVal) {
-  STSRow *   pRow = pIter->pRow;
+  STSRow    *pRow = pIter->pRow;
   SKvRowIdx *pKvIdx = NULL;
   bool       colFound = false;
   col_id_t   kvNCols = tdRowGetNCols(pRow);
@@ -937,9 +937,8 @@ static FORCE_INLINE bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col
     STColumn *pCol = NULL;
     STSchema *pSchema = pIter->pSchema;
     while (pIter->colIdx <= pSchema->numOfCols) {
-      pCol = &pSchema->columns[pIter->colIdx];
+      pCol = &pSchema->columns[pIter->colIdx];  // 1st column of schema is primary TS key
       if (colId == pCol->colId) {
-        ++pIter->colIdx;
         break;
       } else if (colId < pCol->colId) {
         ++pIter->colIdx;
@@ -948,7 +947,8 @@ static FORCE_INLINE bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col
         return false;
       }
     }
-    return tdGetTpRowDataOfCol(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal);
+    tdGetTpRowDataOfCol(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal);
+    ++pIter->colIdx;
   } else if (TD_IS_KV_ROW(pIter->pRow)) {
     return tdGetKvRowValOfColEx(pIter, colId, colType, &pIter->kvIdx, pVal);
   } else {
@@ -1076,7 +1076,7 @@ typedef struct {
 
 typedef struct {
   STSchema *pSchema;
-  STSRow *  pRow;
+  STSRow   *pRow;
 } STSRowReader;
 
 typedef struct {

include/common/ttime.h

@@ -60,8 +60,10 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati
 
 int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth);
 void    deltaToUtcInitOnce();
+char getPrecisionUnit(int32_t precision);
 
 int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision);
+int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit);
 
 void taosFormatUtcTime(char *buf, int32_t bufLen, int64_t time, int32_t precision);
 

include/common/ttokendef.h

@@ -82,21 +82,21 @@
 #define TK_SINGLE_STABLE                   64
 #define TK_STREAM_MODE                     65
 #define TK_RETENTIONS                      66
-#define TK_TABLE                           67
-#define TK_NK_LP                           68
-#define TK_NK_RP                           69
-#define TK_STABLE                          70
-#define TK_ADD                             71
-#define TK_COLUMN                          72
-#define TK_MODIFY                          73
-#define TK_RENAME                          74
-#define TK_TAG                             75
-#define TK_SET                             76
-#define TK_NK_EQ                           77
-#define TK_USING                           78
-#define TK_TAGS                            79
-#define TK_NK_DOT                          80
-#define TK_NK_COMMA                        81
+#define TK_NK_COMMA                        67
+#define TK_TABLE                           68
+#define TK_NK_LP                           69
+#define TK_NK_RP                           70
+#define TK_STABLE                          71
+#define TK_ADD                             72
+#define TK_COLUMN                          73
+#define TK_MODIFY                          74
+#define TK_RENAME                          75
+#define TK_TAG                             76
+#define TK_SET                             77
+#define TK_NK_EQ                           78
+#define TK_USING                           79
+#define TK_TAGS                            80
+#define TK_NK_DOT                          81
 #define TK_COMMENT                         82
 #define TK_BOOL                            83
 #define TK_TINYINT                         84
@@ -138,59 +138,62 @@
 #define TK_INTERVAL                       120
 #define TK_TOPIC                          121
 #define TK_AS                             122
-#define TK_EXPLAIN                        123
-#define TK_ANALYZE                        124
-#define TK_VERBOSE                        125
-#define TK_NK_BOOL                        126
-#define TK_RATIO                          127
-#define TK_NULL                           128
-#define TK_NK_VARIABLE                    129
-#define TK_NK_UNDERLINE                   130
-#define TK_ROWTS                          131
-#define TK_TBNAME                         132
-#define TK_QSTARTTS                       133
-#define TK_QENDTS                         134
-#define TK_WSTARTTS                       135
-#define TK_WENDTS                         136
-#define TK_WDURATION                      137
-#define TK_BETWEEN                        138
-#define TK_IS                             139
-#define TK_NK_LT                          140
-#define TK_NK_GT                          141
-#define TK_NK_LE                          142
-#define TK_NK_GE                          143
-#define TK_NK_NE                          144
-#define TK_MATCH                          145
-#define TK_NMATCH                         146
-#define TK_IN                             147
-#define TK_JOIN                           148
-#define TK_INNER                          149
-#define TK_SELECT                         150
-#define TK_DISTINCT                       151
-#define TK_WHERE                          152
-#define TK_PARTITION                      153
-#define TK_BY                             154
-#define TK_SESSION                        155
-#define TK_STATE_WINDOW                   156
-#define TK_SLIDING                        157
-#define TK_FILL                           158
-#define TK_VALUE                          159
-#define TK_NONE                           160
-#define TK_PREV                           161
-#define TK_LINEAR                         162
-#define TK_NEXT                           163
-#define TK_GROUP                          164
-#define TK_HAVING                         165
-#define TK_ORDER                          166
-#define TK_SLIMIT                         167
-#define TK_SOFFSET                        168
-#define TK_LIMIT                          169
-#define TK_OFFSET                         170
-#define TK_ASC                            171
-#define TK_DESC                           172
-#define TK_NULLS                          173
-#define TK_FIRST                          174
-#define TK_LAST                           175
+#define TK_DESC                           123
+#define TK_DESCRIBE                       124
+#define TK_RESET                          125
+#define TK_QUERY                          126
+#define TK_EXPLAIN                        127
+#define TK_ANALYZE                        128
+#define TK_VERBOSE                        129
+#define TK_NK_BOOL                        130
+#define TK_RATIO                          131
+#define TK_NULL                           132
+#define TK_NK_VARIABLE                    133
+#define TK_NK_UNDERLINE                   134
+#define TK_ROWTS                          135
+#define TK_TBNAME                         136
+#define TK_QSTARTTS                       137
+#define TK_QENDTS                         138
+#define TK_WSTARTTS                       139
+#define TK_WENDTS                         140
+#define TK_WDURATION                      141
+#define TK_BETWEEN                        142
+#define TK_IS                             143
+#define TK_NK_LT                          144
+#define TK_NK_GT                          145
+#define TK_NK_LE                          146
+#define TK_NK_GE                          147
+#define TK_NK_NE                          148
+#define TK_MATCH                          149
+#define TK_NMATCH                         150
+#define TK_IN                             151
+#define TK_JOIN                           152
+#define TK_INNER                          153
+#define TK_SELECT                         154
+#define TK_DISTINCT                       155
+#define TK_WHERE                          156
+#define TK_PARTITION                      157
+#define TK_BY                             158
+#define TK_SESSION                        159
+#define TK_STATE_WINDOW                   160
+#define TK_SLIDING                        161
+#define TK_FILL                           162
+#define TK_VALUE                          163
+#define TK_NONE                           164
+#define TK_PREV                           165
+#define TK_LINEAR                         166
+#define TK_NEXT                           167
+#define TK_GROUP                          168
+#define TK_HAVING                         169
+#define TK_ORDER                          170
+#define TK_SLIMIT                         171
+#define TK_SOFFSET                        172
+#define TK_LIMIT                          173
+#define TK_OFFSET                         174
+#define TK_ASC                            175
+#define TK_NULLS                          176
+#define TK_FIRST                          177
+#define TK_LAST                           178
 
 #define TK_NK_SPACE                       300
 #define TK_NK_COMMENT                     301

include/libs/command/command.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cmdnodes.h"
+#include "tmsg.h"
+#include "plannodes.h"
+
+int32_t qExecCommand(SNode* pStmt, SRetrieveTableRsp** pRsp);
+
+int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp);
+
+

include/libs/nodes/cmdnodes.h

@@ -20,8 +20,14 @@
 extern "C" {
 #endif
 
+#include "query.h"
 #include "querynodes.h"
 
+#define DESCRIBE_RESULT_COLS 4
+#define DESCRIBE_RESULT_FIELD_LEN (TSDB_COL_NAME_LEN - 1 + VARSTR_HEADER_SIZE)
+#define DESCRIBE_RESULT_TYPE_LEN (20 + VARSTR_HEADER_SIZE)
+#define DESCRIBE_RESULT_NOTE_LEN (8 + VARSTR_HEADER_SIZE)
+
 typedef struct SDatabaseOptions {
   ENodeType type;
   int32_t numOfBlocks;
@@ -32,7 +38,9 @@ typedef struct SDatabaseOptions {
   int32_t fsyncPeriod;
   int32_t maxRowsPerBlock;
   int32_t minRowsPerBlock;
-  int32_t keep;
+  int32_t keep0;
+  int32_t keep1;
+  int32_t keep2;
   int32_t precision;
   int32_t quorum;
   int32_t replica;
@@ -70,7 +78,9 @@ typedef struct SAlterDatabaseStmt {
 
 typedef struct STableOptions {
   ENodeType type;
-  int32_t keep;
+  int32_t keep0;
+  int32_t keep1;
+  int32_t keep2;
   int32_t ttl;
   char comments[TSDB_STB_COMMENT_LEN];
   SNodeList* pSma;
@@ -247,6 +257,13 @@ typedef struct SAlterLocalStmt {
   char value[TSDB_DNODE_VALUE_LEN];
 } SAlterLocalStmt;
 
+typedef struct SDescribeStmt {
+  ENodeType type;
+  char dbName[TSDB_DB_NAME_LEN];
+  char tableName[TSDB_TABLE_NAME_LEN];
+  STableMeta* pMeta;
+} SDescribeStmt;
+
 #ifdef __cplusplus
 }
 #endif

include/libs/nodes/nodes.h

@@ -101,6 +101,8 @@ typedef enum ENodeType {
   QUERY_NODE_DROP_TOPIC_STMT,
   QUERY_NODE_ALTER_LOCAL_STMT,
   QUERY_NODE_EXPLAIN_STMT,
+  QUERY_NODE_DESCRIBE_STMT,
+  QUERY_NODE_RESET_QUERY_CACHE_STMT,
   QUERY_NODE_SHOW_DATABASES_STMT,
   QUERY_NODE_SHOW_TABLES_STMT,
   QUERY_NODE_SHOW_STABLES_STMT,
@@ -213,6 +215,8 @@ int32_t nodesStringToNode(const char* pStr, SNode** pNode);
 int32_t nodesListToString(const SNodeList* pList, bool format, char** pStr, int32_t* pLen);
 int32_t nodesStringToList(const char* pStr, SNodeList** pList);
 
+int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len);
+
 #ifdef __cplusplus
 }
 #endif

include/libs/nodes/querynodes.h

@@ -314,6 +314,7 @@ bool nodesIsTimeorderQuery(const SNode* pQuery);
 bool nodesIsTimelineQuery(const SNode* pQuery);
 
 void* nodesGetValueFromNode(SValueNode *pNode);
+char* nodesGetStrValueFromNode(SValueNode *pNode);
 
 #ifdef __cplusplus
 }

include/libs/parser/parser.h

@@ -55,6 +55,7 @@ typedef struct SQuery {
   SArray* pDbList;
   SArray* pTableList;
   bool showRewrite;
+  bool localCmd;
 } SQuery;
 
 int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery);

include/libs/qcom/query.h

@@ -53,6 +53,7 @@ typedef struct SIndexMeta {
 
 } SIndexMeta;
 
+
 /*
  * ASSERT(sizeof(SCTableMeta) == 24)
  * ASSERT(tableType == TSDB_CHILD_TABLE)
@@ -235,6 +236,11 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
     }                                                                    \
   } while (0)
 
+#define QRY_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
+#define QRY_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
+#define QRY_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
+
+
 #ifdef __cplusplus
 }
 #endif

include/libs/wal/wal.h

@@ -157,6 +157,7 @@ typedef struct SWalReadHandle {
   int64_t       curVersion;
   int64_t       capacity;
   int64_t       status;  // if cursor valid
+  TdThreadMutex mutex;
   SWalHead     *pHead;
 } SWalReadHandle;
 #pragma pack(pop)
@@ -191,6 +192,7 @@ int32_t walEndSnapshot(SWal *);
 SWalReadHandle *walOpenReadHandle(SWal *);
 void            walCloseReadHandle(SWalReadHandle *);
 int32_t         walReadWithHandle(SWalReadHandle *pRead, int64_t ver);
+int32_t         walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead);
 
 // deprecated
 #if 0

include/os/os.h

@@ -82,6 +82,7 @@ extern "C" {
 #include "osLz4.h"
 #include "osMath.h"
 #include "osMemory.h"
+#include "osProc.h"
 #include "osRand.h"
 #include "osThread.h"
 #include "osSemaphore.h"

include/os/osMemory.h

@@ -32,6 +32,7 @@ extern "C" {
 void *taosMemoryMalloc(int32_t size);
 void *taosMemoryCalloc(int32_t num, int32_t size);
 void *taosMemoryRealloc(void *ptr, int32_t size);
+void *taosMemoryStrDup(void *ptr);
 void taosMemoryFree(const void *ptr);
 int32_t taosMemorySize(void *ptr);
 

include/os/osProc.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_OS_PROC_H_
+#define _TD_OS_PROC_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t taosNewProc(char **args);
+void    taosWaitProc(int32_t pid);
+void    taosKillProc(int32_t pid);
+bool    taosProcExist(int32_t pid);
+void    taosSetProcName(int32_t argc, char **argv, const char *name);
+void    taosSetProcPath(int32_t argc, char **argv);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_OS_PROC_H_*/

include/os/osShm.h

@@ -29,7 +29,6 @@ typedef struct {
 int32_t taosCreateShm(SShm *pShm, int32_t shmsize) ;
 void    taosDropShm(SShm *pShm);
 int32_t taosAttachShm(SShm *pShm);
-void    taosDetachShm(SShm *pShm);
 
 #ifdef __cplusplus
 }

include/os/osSignal.h

@@ -49,6 +49,8 @@ void taosSetSignal(int32_t signum, FSignalHandler sigfp);
 void taosIgnSignal(int32_t signum);
 void taosDflSignal(int32_t signum);
 
+void taosKillChildOnParentStopped();
+
 #ifdef __cplusplus
 }
 #endif

include/os/osThread.h

@@ -42,103 +42,103 @@ typedef pthread_barrierattr_t TdThreadBarrierAttr;
 // If the error is in a third-party library, place this header file under the third-party library header file.
 // When you want to use this feature, you should find or add the same function in the following section.
 #ifndef ALLOW_FORBID_FUNC
-    // #define pthread_t PTHREAD_T_TYPE_TAOS_FORBID
-    // #define pthread_spinlock_t PTHREAD_SPINLOCK_T_TYPE_TAOS_FORBID
-    // #define pthread_mutex_t PTHREAD_MUTEX_T_TYPE_TAOS_FORBID
-    // #define pthread_mutexattr_t PTHREAD_MUTEXATTR_T_TYPE_TAOS_FORBID
-    // #define pthread_rwlock_t PTHREAD_RWLOCK_T_TYPE_TAOS_FORBID
-    // #define pthread_attr_t PTHREAD_ATTR_T_TYPE_TAOS_FORBID
-    // #define pthread_once_t PTHREAD_ONCE_T_TYPE_TAOS_FORBID
-    // #define pthread_rwlockattr_t PTHREAD_RWLOCKATTR_T_TYPE_TAOS_FORBID
-    // #define pthread_cond_t PTHREAD_COND_T_TYPE_TAOS_FORBID
-    // #define pthread_condattr_t PTHREAD_CONDATTR_T_TYPE_TAOS_FORBID
-    // #define pthread_key_t PTHREAD_KEY_T_TYPE_TAOS_FORBID
-    // #define pthread_barrier_t PTHREAD_BARRIER_T_TYPE_TAOS_FORBID
-    // #define pthread_barrierattr_t PTHREAD_BARRIERATTR_T_TYPE_TAOS_FORBID
-    // #define pthread_create PTHREAD_CREATE_FUNC_TAOS_FORBID
-    // #define pthread_attr_destroy PTHREAD_ATTR_DESTROY_FUNC_TAOS_FORBID
-    // #define pthread_attr_getdetachstate PTHREAD_ATTR_GETDETACHSTATE_FUNC_TAOS_FORBID
-    // #define pthread_attr_getinheritsched PTHREAD_ATTR_GETINHERITSCHED_FUNC_TAOS_FORBID
-    // #define pthread_attr_getschedparam PTHREAD_ATTR_GETSCHEDPARAM_FUNC_TAOS_FORBID
-    // #define pthread_attr_getschedpolicy PTHREAD_ATTR_GETSCHEDPOLICY_FUNC_TAOS_FORBID
-    // #define pthread_attr_getscope PTHREAD_ATTR_GETSCOPE_FUNC_TAOS_FORBID
-    // #define pthread_attr_getstacksize PTHREAD_ATTR_GETSTACKSIZE_FUNC_TAOS_FORBID
-    // #define pthread_attr_init PTHREAD_ATTR_INIT_FUNC_TAOS_FORBID
-    // #define pthread_attr_setdetachstate PTHREAD_ATTR_SETDETACHSTATE_FUNC_TAOS_FORBID
-    // #define pthread_attr_setinheritsched PTHREAD_ATTR_SETINHERITSCHED_FUNC_TAOS_FORBID
-    // #define pthread_attr_setschedparam PTHREAD_ATTR_SETSCHEDPARAM_FUNC_TAOS_FORBID
-    // #define pthread_attr_setschedpolicy PTHREAD_ATTR_SETSCHEDPOLICY_FUNC_TAOS_FORBID
-    // #define pthread_attr_setscope PTHREAD_ATTR_SETSCOPE_FUNC_TAOS_FORBID
-    // #define pthread_attr_setstacksize PTHREAD_ATTR_SETSTACKSIZE_FUNC_TAOS_FORBID
-    // #define pthread_barrier_destroy PTHREAD_BARRIER_DESTROY_FUNC_TAOS_FORBID
-    // #define pthread_barrier_init PTHREAD_BARRIER_INIT_FUNC_TAOS_FORBID
-    // #define pthread_barrier_wait PTHREAD_BARRIER_WAIT_FUNC_TAOS_FORBID
-    // #define pthread_barrierattr_destroy PTHREAD_BARRIERATTR_DESTROY_FUNC_TAOS_FORBID
-    // #define pthread_barrierattr_getpshared PTHREAD_BARRIERATTR_GETPSHARED_FUNC_TAOS_FORBID
-    // #define pthread_barrierattr_init PTHREAD_BARRIERATTR_INIT_FUNC_TAOS_FORBID
-    // #define pthread_barrierattr_setpshared PTHREAD_BARRIERATTR_SETPSHARED_FUNC_TAOS_FORBID
-    // #define pthread_cancel PTHREAD_CANCEL_FUNC_TAOS_FORBID
-    // #define pthread_cond_destroy PTHREAD_COND_DESTROY_FUNC_TAOS_FORBID
-    // #define pthread_cond_init PTHREAD_COND_INIT_FUNC_TAOS_FORBID
-    // #define pthread_cond_signal PTHREAD_COND_SIGNAL_FUNC_TAOS_FORBID
-    // #define pthread_cond_broadcast PTHREAD_COND_BROADCAST_FUNC_TAOS_FORBID
-    // #define pthread_cond_wait PTHREAD_COND_WAIT_FUNC_TAOS_FORBID
-    // #define pthread_cond_timedwait PTHREAD_COND_TIMEDWAIT_FUNC_TAOS_FORBID
-    // #define pthread_condattr_destroy PTHREAD_CONDATTR_DESTROY_FUNC_TAOS_FORBID
-    // #define pthread_condattr_getpshared PTHREAD_CONDATTR_GETPSHARED_FUNC_TAOS_FORBID
-    // #define pthread_condattr_init PTHREAD_CONDATTR_INIT_FUNC_TAOS_FORBID
-    // #define pthread_condattr_setpshared PTHREAD_CONDATTR_SETPSHARED_FUNC_TAOS_FORBID
-    // #define pthread_detach PTHREAD_DETACH_FUNC_TAOS_FORBID
-    // #define pthread_equal PTHREAD_EQUAL_FUNC_TAOS_FORBID
-    // #define pthread_exit PTHREAD_EXIT_FUNC_TAOS_FORBID
-    // #define pthread_getschedparam PTHREAD_GETSCHEDPARAM_FUNC_TAOS_FORBID
-    // #define pthread_getspecific PTHREAD_GETSPECIFIC_FUNC_TAOS_FORBID
-    // #define pthread_join PTHREAD_JOIN_FUNC_TAOS_FORBID
-    // #define pthread_key_create PTHREAD_KEY_CREATE_FUNC_TAOS_FORBID
-    // #define pthread_key_delete PTHREAD_KEY_DELETE_FUNC_TAOS_FORBID
-    // #define pthread_kill PTHREAD_KILL_FUNC_TAOS_FORBID
-    // #define pthread_mutex_consistent PTHREAD_MUTEX_CONSISTENT_FUNC_TAOS_FORBID
-    // #define pthread_mutex_destroy PTHREAD_MUTEX_DESTROY_FUNC_TAOS_FORBID
-    // #define pthread_mutex_init PTHREAD_MUTEX_INIT_FUNC_TAOS_FORBID
-    // #define pthread_mutex_lock PTHREAD_MUTEX_LOCK_FUNC_TAOS_FORBID
-    // #define pthread_mutex_timedlock PTHREAD_MUTEX_TIMEDLOCK_FUNC_TAOS_FORBID
-    // #define pthread_mutex_trylock PTHREAD_MUTEX_TRYLOCK_FUNC_TAOS_FORBID
-    // #define pthread_mutex_unlock PTHREAD_MUTEX_UNLOCK_FUNC_TAOS_FORBID
-    // #define pthread_mutexattr_destroy PTHREAD_MUTEXATTR_DESTROY_FUNC_TAOS_FORBID
-    // #define pthread_mutexattr_getpshared PTHREAD_MUTEXATTR_GETPSHARED_FUNC_TAOS_FORBID
-    // #define pthread_mutexattr_getrobust PTHREAD_MUTEXATTR_GETROBUST_FUNC_TAOS_FORBID
-    // #define pthread_mutexattr_gettype PTHREAD_MUTEXATTR_GETTYPE_FUNC_TAOS_FORBID
-    // #define pthread_mutexattr_init PTHREAD_MUTEXATTR_INIT_FUNC_TAOS_FORBID
-    // #define pthread_mutexattr_setpshared PTHREAD_MUTEXATTR_SETPSHARED_FUNC_TAOS_FORBID
-    // #define pthread_mutexattr_setrobust PTHREAD_MUTEXATTR_SETROBUST_FUNC_TAOS_FORBID
-    // #define pthread_mutexattr_settype PTHREAD_MUTEXATTR_SETTYPE_FUNC_TAOS_FORBID
-    // #define pthread_once PTHREAD_ONCE_FUNC_TAOS_FORBID
-    // #define pthread_rwlock_destroy PTHREAD_RWLOCK_DESTROY_FUNC_TAOS_FORBID
-    // #define pthread_rwlock_init PTHREAD_RWLOCK_INIT_FUNC_TAOS_FORBID
-    // #define pthread_rwlock_rdlock PTHREAD_RWLOCK_RDLOCK_FUNC_TAOS_FORBID
-    // #define pthread_rwlock_timedrdlock PTHREAD_RWLOCK_TIMEDRDLOCK_FUNC_TAOS_FORBID
-    // #define pthread_rwlock_timedwrlock PTHREAD_RWLOCK_TIMEDWRLOCK_FUNC_TAOS_FORBID
-    // #define pthread_rwlock_tryrdlock PTHREAD_RWLOCK_TRYRDLOCK_FUNC_TAOS_FORBID
-    // #define pthread_rwlock_trywrlock PTHREAD_RWLOCK_TRYWRLOCK_FUNC_TAOS_FORBID
-    // #define pthread_rwlock_unlock PTHREAD_RWLOCK_UNLOCK_FUNC_TAOS_FORBID
-    // #define pthread_rwlock_wrlock PTHREAD_RWLOCK_WRLOCK_FUNC_TAOS_FORBID
-    // #define pthread_rwlockattr_destroy PTHREAD_RWLOCKATTR_DESTROY_FUNC_TAOS_FORBID
-    // #define pthread_rwlockattr_getpshared PTHREAD_RWLOCKATTR_GETPSHARED_FUNC_TAOS_FORBID
-    // #define pthread_rwlockattr_init PTHREAD_RWLOCKATTR_INIT_FUNC_TAOS_FORBID
-    // #define pthread_rwlockattr_setpshared PTHREAD_RWLOCKATTR_SETPSHARED_FUNC_TAOS_FORBID
-    // #define pthread_self PTHREAD_SELF_FUNC_TAOS_FORBID
-    // #define pthread_setcancelstate PTHREAD_SETCANCELSTATE_FUNC_TAOS_FORBID
-    // #define pthread_setcanceltype PTHREAD_SETCANCELTYPE_FUNC_TAOS_FORBID
-    // #define pthread_setschedparam PTHREAD_SETSCHEDPARAM_FUNC_TAOS_FORBID
-    // #define pthread_setspecific PTHREAD_SETSPECIFIC_FUNC_TAOS_FORBID
-    // #define pthread_spin_destroy PTHREAD_SPIN_DESTROY_FUNC_TAOS_FORBID
-    // #define pthread_spin_init PTHREAD_SPIN_INIT_FUNC_TAOS_FORBID
-    // #define pthread_spin_lock PTHREAD_SPIN_LOCK_FUNC_TAOS_FORBID
-    // #define pthread_spin_trylock PTHREAD_SPIN_TRYLOCK_FUNC_TAOS_FORBID
-    // #define pthread_spin_unlock PTHREAD_SPIN_UNLOCK_FUNC_TAOS_FORBID
-    // #define pthread_testcancel PTHREAD_TESTCANCEL_FUNC_TAOS_FORBID
-    // #define pthread_sigmask PTHREAD_SIGMASK_FUNC_TAOS_FORBID
-    // #define sigwait SIGWAIT_FUNC_TAOS_FORBID
+    #define pthread_t PTHREAD_T_TYPE_TAOS_FORBID
+    #define pthread_spinlock_t PTHREAD_SPINLOCK_T_TYPE_TAOS_FORBID
+    #define pthread_mutex_t PTHREAD_MUTEX_T_TYPE_TAOS_FORBID
+    #define pthread_mutexattr_t PTHREAD_MUTEXATTR_T_TYPE_TAOS_FORBID
+    #define pthread_rwlock_t PTHREAD_RWLOCK_T_TYPE_TAOS_FORBID
+    #define pthread_attr_t PTHREAD_ATTR_T_TYPE_TAOS_FORBID
+    #define pthread_once_t PTHREAD_ONCE_T_TYPE_TAOS_FORBID
+    #define pthread_rwlockattr_t PTHREAD_RWLOCKATTR_T_TYPE_TAOS_FORBID
+    #define pthread_cond_t PTHREAD_COND_T_TYPE_TAOS_FORBID
+    #define pthread_condattr_t PTHREAD_CONDATTR_T_TYPE_TAOS_FORBID
+    #define pthread_key_t PTHREAD_KEY_T_TYPE_TAOS_FORBID
+    #define pthread_barrier_t PTHREAD_BARRIER_T_TYPE_TAOS_FORBID
+    #define pthread_barrierattr_t PTHREAD_BARRIERATTR_T_TYPE_TAOS_FORBID
+    #define pthread_create PTHREAD_CREATE_FUNC_TAOS_FORBID
+    #define pthread_attr_destroy PTHREAD_ATTR_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_attr_getdetachstate PTHREAD_ATTR_GETDETACHSTATE_FUNC_TAOS_FORBID
+    #define pthread_attr_getinheritsched PTHREAD_ATTR_GETINHERITSCHED_FUNC_TAOS_FORBID
+    #define pthread_attr_getschedparam PTHREAD_ATTR_GETSCHEDPARAM_FUNC_TAOS_FORBID
+    #define pthread_attr_getschedpolicy PTHREAD_ATTR_GETSCHEDPOLICY_FUNC_TAOS_FORBID
+    #define pthread_attr_getscope PTHREAD_ATTR_GETSCOPE_FUNC_TAOS_FORBID
+    #define pthread_attr_getstacksize PTHREAD_ATTR_GETSTACKSIZE_FUNC_TAOS_FORBID
+    #define pthread_attr_init PTHREAD_ATTR_INIT_FUNC_TAOS_FORBID
+    #define pthread_attr_setdetachstate PTHREAD_ATTR_SETDETACHSTATE_FUNC_TAOS_FORBID
+    #define pthread_attr_setinheritsched PTHREAD_ATTR_SETINHERITSCHED_FUNC_TAOS_FORBID
+    #define pthread_attr_setschedparam PTHREAD_ATTR_SETSCHEDPARAM_FUNC_TAOS_FORBID
+    #define pthread_attr_setschedpolicy PTHREAD_ATTR_SETSCHEDPOLICY_FUNC_TAOS_FORBID
+    #define pthread_attr_setscope PTHREAD_ATTR_SETSCOPE_FUNC_TAOS_FORBID
+    #define pthread_attr_setstacksize PTHREAD_ATTR_SETSTACKSIZE_FUNC_TAOS_FORBID
+    #define pthread_barrier_destroy PTHREAD_BARRIER_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_barrier_init PTHREAD_BARRIER_INIT_FUNC_TAOS_FORBID
+    #define pthread_barrier_wait PTHREAD_BARRIER_WAIT_FUNC_TAOS_FORBID
+    #define pthread_barrierattr_destroy PTHREAD_BARRIERATTR_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_barrierattr_getpshared PTHREAD_BARRIERATTR_GETPSHARED_FUNC_TAOS_FORBID
+    #define pthread_barrierattr_init PTHREAD_BARRIERATTR_INIT_FUNC_TAOS_FORBID
+    #define pthread_barrierattr_setpshared PTHREAD_BARRIERATTR_SETPSHARED_FUNC_TAOS_FORBID
+    #define pthread_cancel PTHREAD_CANCEL_FUNC_TAOS_FORBID
+    #define pthread_cond_destroy PTHREAD_COND_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_cond_init PTHREAD_COND_INIT_FUNC_TAOS_FORBID
+    #define pthread_cond_signal PTHREAD_COND_SIGNAL_FUNC_TAOS_FORBID
+    #define pthread_cond_broadcast PTHREAD_COND_BROADCAST_FUNC_TAOS_FORBID
+    #define pthread_cond_wait PTHREAD_COND_WAIT_FUNC_TAOS_FORBID
+    #define pthread_cond_timedwait PTHREAD_COND_TIMEDWAIT_FUNC_TAOS_FORBID
+    #define pthread_condattr_destroy PTHREAD_CONDATTR_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_condattr_getpshared PTHREAD_CONDATTR_GETPSHARED_FUNC_TAOS_FORBID
+    #define pthread_condattr_init PTHREAD_CONDATTR_INIT_FUNC_TAOS_FORBID
+    #define pthread_condattr_setpshared PTHREAD_CONDATTR_SETPSHARED_FUNC_TAOS_FORBID
+    #define pthread_detach PTHREAD_DETACH_FUNC_TAOS_FORBID
+    #define pthread_equal PTHREAD_EQUAL_FUNC_TAOS_FORBID
+    #define pthread_exit PTHREAD_EXIT_FUNC_TAOS_FORBID
+    #define pthread_getschedparam PTHREAD_GETSCHEDPARAM_FUNC_TAOS_FORBID
+    #define pthread_getspecific PTHREAD_GETSPECIFIC_FUNC_TAOS_FORBID
+    #define pthread_join PTHREAD_JOIN_FUNC_TAOS_FORBID
+    #define pthread_key_create PTHREAD_KEY_CREATE_FUNC_TAOS_FORBID
+    #define pthread_key_delete PTHREAD_KEY_DELETE_FUNC_TAOS_FORBID
+    #define pthread_kill PTHREAD_KILL_FUNC_TAOS_FORBID
+    #define pthread_mutex_consistent PTHREAD_MUTEX_CONSISTENT_FUNC_TAOS_FORBID
+    #define pthread_mutex_destroy PTHREAD_MUTEX_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_mutex_init PTHREAD_MUTEX_INIT_FUNC_TAOS_FORBID
+    #define pthread_mutex_lock PTHREAD_MUTEX_LOCK_FUNC_TAOS_FORBID
+    #define pthread_mutex_timedlock PTHREAD_MUTEX_TIMEDLOCK_FUNC_TAOS_FORBID
+    #define pthread_mutex_trylock PTHREAD_MUTEX_TRYLOCK_FUNC_TAOS_FORBID
+    #define pthread_mutex_unlock PTHREAD_MUTEX_UNLOCK_FUNC_TAOS_FORBID
+    #define pthread_mutexattr_destroy PTHREAD_MUTEXATTR_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_mutexattr_getpshared PTHREAD_MUTEXATTR_GETPSHARED_FUNC_TAOS_FORBID
+    #define pthread_mutexattr_getrobust PTHREAD_MUTEXATTR_GETROBUST_FUNC_TAOS_FORBID
+    #define pthread_mutexattr_gettype PTHREAD_MUTEXATTR_GETTYPE_FUNC_TAOS_FORBID
+    #define pthread_mutexattr_init PTHREAD_MUTEXATTR_INIT_FUNC_TAOS_FORBID
+    #define pthread_mutexattr_setpshared PTHREAD_MUTEXATTR_SETPSHARED_FUNC_TAOS_FORBID
+    #define pthread_mutexattr_setrobust PTHREAD_MUTEXATTR_SETROBUST_FUNC_TAOS_FORBID
+    #define pthread_mutexattr_settype PTHREAD_MUTEXATTR_SETTYPE_FUNC_TAOS_FORBID
+    #define pthread_once PTHREAD_ONCE_FUNC_TAOS_FORBID
+    #define pthread_rwlock_destroy PTHREAD_RWLOCK_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_rwlock_init PTHREAD_RWLOCK_INIT_FUNC_TAOS_FORBID
+    #define pthread_rwlock_rdlock PTHREAD_RWLOCK_RDLOCK_FUNC_TAOS_FORBID
+    #define pthread_rwlock_timedrdlock PTHREAD_RWLOCK_TIMEDRDLOCK_FUNC_TAOS_FORBID
+    #define pthread_rwlock_timedwrlock PTHREAD_RWLOCK_TIMEDWRLOCK_FUNC_TAOS_FORBID
+    #define pthread_rwlock_tryrdlock PTHREAD_RWLOCK_TRYRDLOCK_FUNC_TAOS_FORBID
+    #define pthread_rwlock_trywrlock PTHREAD_RWLOCK_TRYWRLOCK_FUNC_TAOS_FORBID
+    #define pthread_rwlock_unlock PTHREAD_RWLOCK_UNLOCK_FUNC_TAOS_FORBID
+    #define pthread_rwlock_wrlock PTHREAD_RWLOCK_WRLOCK_FUNC_TAOS_FORBID
+    #define pthread_rwlockattr_destroy PTHREAD_RWLOCKATTR_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_rwlockattr_getpshared PTHREAD_RWLOCKATTR_GETPSHARED_FUNC_TAOS_FORBID
+    #define pthread_rwlockattr_init PTHREAD_RWLOCKATTR_INIT_FUNC_TAOS_FORBID
+    #define pthread_rwlockattr_setpshared PTHREAD_RWLOCKATTR_SETPSHARED_FUNC_TAOS_FORBID
+    #define pthread_self PTHREAD_SELF_FUNC_TAOS_FORBID
+    #define pthread_setcancelstate PTHREAD_SETCANCELSTATE_FUNC_TAOS_FORBID
+    #define pthread_setcanceltype PTHREAD_SETCANCELTYPE_FUNC_TAOS_FORBID
+    #define pthread_setschedparam PTHREAD_SETSCHEDPARAM_FUNC_TAOS_FORBID
+    #define pthread_setspecific PTHREAD_SETSPECIFIC_FUNC_TAOS_FORBID
+    #define pthread_spin_destroy PTHREAD_SPIN_DESTROY_FUNC_TAOS_FORBID
+    #define pthread_spin_init PTHREAD_SPIN_INIT_FUNC_TAOS_FORBID
+    #define pthread_spin_lock PTHREAD_SPIN_LOCK_FUNC_TAOS_FORBID
+    #define pthread_spin_trylock PTHREAD_SPIN_TRYLOCK_FUNC_TAOS_FORBID
+    #define pthread_spin_unlock PTHREAD_SPIN_UNLOCK_FUNC_TAOS_FORBID
+    #define pthread_testcancel PTHREAD_TESTCANCEL_FUNC_TAOS_FORBID
+    #define pthread_sigmask PTHREAD_SIGMASK_FUNC_TAOS_FORBID
+    #define sigwait SIGWAIT_FUNC_TAOS_FORBID
 #endif
 
 int32_t taosThreadCreate(TdThread * tid, const TdThreadAttr * attr, void *(*start)(void *), void *arg);

include/os/osTime.h

@@ -46,6 +46,14 @@ extern "C" {
 #define MILLISECOND_PER_DAY    (MILLISECOND_PER_HOUR * 24)
 #define MILLISECOND_PER_WEEK   (MILLISECOND_PER_DAY * 7)
 
+#define NANOSECOND_PER_USEC   (1000L)
+#define NANOSECOND_PER_MSEC   (1000000L)
+#define NANOSECOND_PER_SEC    (1000000000L)
+#define NANOSECOND_PER_MINUTE (NANOSECOND_PER_SEC * 60)
+#define NANOSECOND_PER_HOUR   (NANOSECOND_PER_MINUTE * 60)
+#define NANOSECOND_PER_DAY    (NANOSECOND_PER_HOUR * 24)
+#define NANOSECOND_PER_WEEK   (NANOSECOND_PER_DAY * 7)
+
 int32_t taosGetTimeOfDay(struct timeval *tv);
 
 //@return timestamp in second

include/util/taoserror.h

@@ -78,6 +78,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_CFG_NOT_FOUND                 TAOS_DEF_ERROR_CODE(0, 0x010C)
 #define TSDB_CODE_INVALID_CFG                   TAOS_DEF_ERROR_CODE(0, 0x010D)
 #define TSDB_CODE_OUT_OF_SHM_MEM                TAOS_DEF_ERROR_CODE(0, 0x010E)
+#define TSDB_CODE_INVALID_SHM_ID                TAOS_DEF_ERROR_CODE(0, 0x010F)
 #define TSDB_CODE_REF_NO_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0110)
 #define TSDB_CODE_REF_FULL                      TAOS_DEF_ERROR_CODE(0, 0x0111)
 #define TSDB_CODE_REF_ID_REMOVED                TAOS_DEF_ERROR_CODE(0, 0x0112)
@@ -478,6 +479,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INVALID_ENDPOINT          TAOS_DEF_ERROR_CODE(0, 0x2613)
 #define TSDB_CODE_PAR_EXPRIE_STATEMENT          TAOS_DEF_ERROR_CODE(0, 0x2614)
 #define TSDB_CODE_PAR_INTERVAL_VALUE_TOO_SMALL  TAOS_DEF_ERROR_CODE(0, 0x2615)
+#define TSDB_CODE_PAR_DB_NOT_SPECIFIED          TAOS_DEF_ERROR_CODE(0, 0x2616)
+#define TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME   TAOS_DEF_ERROR_CODE(0, 0x2617)
+#define TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR  TAOS_DEF_ERROR_CODE(0, 0x2618)
 
 #ifdef __cplusplus
 }

include/util/tdef.h

@@ -224,6 +224,7 @@ typedef enum ELogicConditionType {
 
 #define TSDB_APP_NAME_LEN    TSDB_UNI_LEN
 #define TSDB_STB_COMMENT_LEN 1024
+
 /**
  *  In some scenarios uint16_t (0~65535) is used to store the row len.
  *  - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header.
@@ -303,13 +304,13 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_TOTAL_BLOCKS     10000
 #define TSDB_DEFAULT_TOTAL_BLOCKS 6
 
-#define TSDB_MIN_DAYS_PER_FILE     1
-#define TSDB_MAX_DAYS_PER_FILE     3650
-#define TSDB_DEFAULT_DAYS_PER_FILE 10
+#define TSDB_MIN_DAYS_PER_FILE     (1 * 1440)    // unit minute
+#define TSDB_MAX_DAYS_PER_FILE     (3650 * 1440)
+#define TSDB_DEFAULT_DAYS_PER_FILE (10 * 1440)
 
-#define TSDB_MIN_KEEP     1       // data in db to be reserved.
-#define TSDB_MAX_KEEP     365000  // data in db to be reserved.
-#define TSDB_DEFAULT_KEEP 3650    // ten years
+#define TSDB_MIN_KEEP     (1 * 1440)       // data in db to be reserved. unit minute
+#define TSDB_MAX_KEEP     (365000 * 1440) // data in db to be reserved.
+#define TSDB_DEFAULT_KEEP (3650 * 1440)    // ten years
 
 #define TSDB_MIN_MIN_ROW_FBLOCK     10
 #define TSDB_MAX_MIN_ROW_FBLOCK     1000
@@ -327,7 +328,7 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_FSYNC_PERIOD     180000  // millisecond
 #define TSDB_DEFAULT_FSYNC_PERIOD 3000    // three second
 
-#define TSDB_MIN_WAL_LEVEL     0
+#define TSDB_MIN_WAL_LEVEL     1
 #define TSDB_MAX_WAL_LEVEL     2
 #define TSDB_DEFAULT_WAL_LEVEL 1
 
@@ -388,6 +389,7 @@ typedef enum ELogicConditionType {
 #define TSDB_DEFAULT_EXPLAIN_RATIO   0.001
 
 #define TSDB_EXPLAIN_RESULT_ROW_SIZE 1024
+#define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY PLAN"
 
 #define TSDB_MAX_JOIN_TABLE_NUM 10
 #define TSDB_MAX_UNION_CLAUSE   5
@@ -479,6 +481,9 @@ enum {
 #define QND_VGID 1
 #define VND_VGID 0
 
+#define MAX_NUM_STR_SIZE 40
+
+
 #ifdef __cplusplus
 }
 #endif

include/util/tprocess.h

@@ -32,29 +32,25 @@ typedef void *(*ProcConsumeFp)(void *pParent, void *pHead, int16_t headLen, void
                                ProcFuncType ftype);
 
 typedef struct {
-  int32_t       childQueueSize;
   ProcConsumeFp childConsumeFp;
   ProcMallocFp  childMallocHeadFp;
   ProcFreeFp    childFreeHeadFp;
   ProcMallocFp  childMallocBodyFp;
   ProcFreeFp    childFreeBodyFp;
-  int32_t       parentQueueSize;
   ProcConsumeFp parentConsumeFp;
-  ProcMallocFp  parentdMallocHeadFp;
+  ProcMallocFp  parentMallocHeadFp;
   ProcFreeFp    parentFreeHeadFp;
   ProcMallocFp  parentMallocBodyFp;
   ProcFreeFp    parentFreeBodyFp;
-  bool          testFlag;
+  SShm          shm;
   void         *pParent;
   const char   *name;
+  bool          isChild;
 } SProcCfg;
 
 SProcObj *taosProcInit(const SProcCfg *pCfg);
 void      taosProcCleanup(SProcObj *pProc);
 int32_t   taosProcRun(SProcObj *pProc);
-void      taosProcStop(SProcObj *pProc);
-bool      taosProcIsChild(SProcObj *pProc);
-int32_t   taosProcChildId(SProcObj *pProc);
 int32_t   taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
                               ProcFuncType ftype);
 int32_t   taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,

source/client/CMakeLists.txt

@@ -8,7 +8,7 @@ target_include_directories(
 target_link_libraries(
     taos
     INTERFACE api
-    PRIVATE os util common transport nodes parser planner catalog scheduler function qcom
+    PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom
 )
 
 if(${BUILD_TEST})

source/client/src/clientImpl.c

 
 #include "clientInt.h"
 #include "clientLog.h"
+#include "command.h"
 #include "scheduler.h"
 #include "tdatablock.h"
 #include "tdef.h"
@@ -170,7 +171,21 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery) {
   return code;
 }
 
+int32_t execLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
+  SRetrieveTableRsp* pRsp = NULL;
+  int32_t code = qExecCommand(pQuery->pRoot, &pRsp);
+  if (TSDB_CODE_SUCCESS == code && NULL != pRsp) {
+    code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp);
+  }
+  return code;
+}
+
 int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
+  // drop table if exists not_exists_table
+  if (NULL == pQuery->pCmdMsg) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   SCmdMsgInfo* pMsgInfo = pQuery->pCmdMsg;
   pRequest->type = pMsgInfo->msgType;
   pRequest->body.requestMsg = (SDataBuf){.pData = pMsgInfo->pMsg, .len = pMsgInfo->msgLen, .handle = NULL};
@@ -259,7 +274,9 @@ SRequestObj* execQueryImpl(STscObj* pTscObj, const char* sql, int sqlLen) {
   CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
   CHECK_CODE_GOTO(parseSql(pRequest, false, &pQuery), _return);
 
-  if (pQuery->directRpc) {
+  if (pQuery->localCmd) {
+    CHECK_CODE_GOTO(execLocalCmd(pRequest, pQuery), _return);
+  } else if (pQuery->directRpc) {
     CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return);
   } else {
     CHECK_CODE_GOTO(getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList), _return);
@@ -464,9 +481,11 @@ static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) {
   taosMemoryFreeClear(pMsgBody->msgInfo.pData);
   taosMemoryFreeClear(pMsgBody);
 }
+
 bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType) {
   return msgType == TDMT_VND_QUERY_RSP || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP || msgType == TDMT_VND_QUERY_HEARTBEAT_RSP;
 }
+
 void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
   SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->ahandle;
   assert(pMsg->ahandle != NULL);
@@ -647,6 +666,11 @@ void* doFetchRow(SRequestObj* pRequest, bool setupOneRowPtr) {
       }
     }
 
+    if (pResultInfo->completed) {
+      pResultInfo->numOfRows = 0;
+      return NULL;
+    }
+
     SMsgSendInfo* body = buildMsgInfoImpl(pRequest);
 
     int64_t  transporterId = 0;

source/client/src/tmq.c

@@ -255,6 +255,7 @@ void tmqClearUnhandleMsg(tmq_t* tmq) {
       break;
   }
 
+  msg = NULL;
   taosReadAllQitems(tmq->mqueue, tmq->qall);
   while (1) {
     taosGetQitem(tmq->qall, (void**)&msg);
@@ -787,7 +788,7 @@ void tmqShowMsg(tmq_message_t* tmq_message) {
   static bool noPrintSchema;
   char        pBuf[128];
   SMqPollRsp* pRsp = &tmq_message->msg;
-  int32_t     colNum = pRsp->schema->nCols;
+  int32_t     colNum = 2;
   if (!noPrintSchema) {
     printf("|");
     for (int32_t i = 0; i < colNum; i++) {
@@ -838,6 +839,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
   int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch;
   int32_t tmqEpoch = atomic_load_32(&tmq->epoch);
   if (msgEpoch < tmqEpoch) {
+    /*printf("discard rsp epoch %d, current epoch %d\n", msgEpoch, tmqEpoch);*/
     tsem_post(&tmq->rspSem);
     tscWarn("discard rsp epoch %d, current epoch %d", msgEpoch, tmqEpoch);
     return 0;
@@ -886,6 +888,9 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
     goto WRITE_QUEUE_FAIL;
   }
 
+  tscError("tmq recv poll: vg %d, req offset %ld, rsp offset %ld", pParam->pVg->vgId, pRsp->msg.reqOffset,
+           pRsp->msg.rspOffset);
+
   pRsp->vg = pParam->pVg;
   taosWriteQitem(tmq->mqueue, pRsp);
   atomic_add_fetch_32(&tmq->readyRequest, 1);
@@ -902,6 +907,7 @@ WRITE_QUEUE_FAIL:
 
 bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
   /*printf("call update ep %d\n", epoch);*/
+  /*printf("tmq update ep epoch %d to epoch %d\n", tmq->epoch, epoch);*/
   bool    set = false;
   int32_t topicNumGet = taosArrayGetSize(pRsp->topics);
   char    vgKey[TSDB_TOPIC_FNAME_LEN + 22];
@@ -932,6 +938,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
         for (int32_t k = 0; k < vgNumCur; k++) {
           SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, k);
           sprintf(vgKey, "%s:%d", topic.topicName, pVgCur->vgId);
+          /*printf("epoch %d vg %d build %s\n", epoch, pVgCur->vgId, vgKey);*/
           taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t));
         }
         break;
@@ -945,9 +952,12 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
       sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId);
       int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey));
       int64_t  offset = pVgEp->offset;
+      /*printf("epoch %d vg %d offset og to %ld\n", epoch, pVgEp->vgId, offset);*/
       if (pOffset != NULL) {
         offset = *pOffset;
+        /*printf("epoch %d vg %d found %s\n", epoch, pVgEp->vgId, vgKey);*/
       }
+      /*printf("epoch %d vg %d offset set to %ld\n", epoch, pVgEp->vgId, offset);*/
       SMqClientVg clientVg = {
           .pollCnt = 0,
           .currentOffset = offset,
@@ -1195,6 +1205,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
       SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
       int32_t      vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
       if (vgStatus != TMQ_VG_STATUS__IDLE) {
+        /*printf("skip vg %d\n", pVg->vgId);*/
         continue;
       }
       SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
@@ -1238,6 +1249,8 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
       int64_t transporterId = 0;
       /*printf("send poll\n");*/
       atomic_add_fetch_32(&tmq->waitingRequest, 1);
+      /*tscDebug("tmq send poll: vg %d, req offset %ld", pVg->vgId, pVg->currentOffset);*/
+      /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/
       asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
       pVg->pollCnt++;
       tmq->pollCnt++;

source/common/src/tmsg.c

@@ -1629,6 +1629,7 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
   if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->quorum) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->replications) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -1650,6 +1651,7 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
   if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->quorum) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->replications) < 0) return -1;
   tEndDecode(&decoder);
 
   tCoderClear(&decoder);

source/common/src/ttime.c

@@ -361,6 +361,18 @@ int32_t parseLocaltimeDst(char* timestr, int64_t* time, int32_t timePrec) {
   return 0;
 }
 
+char getPrecisionUnit(int32_t precision) {
+  static char units[3] = {TIME_UNIT_MILLISECOND, TIME_UNIT_MICROSECOND, TIME_UNIT_NANOSECOND};
+  switch (precision) {
+    case TSDB_TIME_PRECISION_MILLI:
+    case TSDB_TIME_PRECISION_MICRO:
+    case TSDB_TIME_PRECISION_NANO:
+      return units[precision];
+    default:
+      return 0;
+  }
+}
+
 int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision) {
   assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO ||
          fromPrecision == TSDB_TIME_PRECISION_NANO);
@@ -370,6 +382,33 @@ int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrec
   return (int64_t)((double)time * factors[fromPrecision][toPrecision]);
 }
 
+int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char toUnit) {
+  assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO ||
+         fromPrecision == TSDB_TIME_PRECISION_NANO);
+  static double factors[3] = {1000000., 1000., 1.};
+  switch (toUnit) {
+    case 's':
+      return time * factors[fromPrecision] / NANOSECOND_PER_SEC;
+    case 'm':
+      return time * factors[fromPrecision] / NANOSECOND_PER_MINUTE;
+    case 'h':
+      return time * factors[fromPrecision] / NANOSECOND_PER_HOUR;
+    case 'd':
+      return time * factors[fromPrecision] / NANOSECOND_PER_DAY;
+    case 'w':
+      return time * factors[fromPrecision] / NANOSECOND_PER_WEEK;
+    case 'a':
+      return time * factors[fromPrecision] / NANOSECOND_PER_MSEC;
+    case 'u':
+      return time * factors[fromPrecision] / NANOSECOND_PER_USEC;
+    case 'b':
+      return time * factors[fromPrecision];
+    default: {
+      return -1;
+    }
+  }  
+}
+
 static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) {
   switch (unit) {
     case 's':

source/dnode/mgmt/dm/src/dmFile.c

@@ -130,7 +130,7 @@ int32_t dmReadFile(SDnodeMgmt *pMgmt) {
   }
 
   code = 0;
-  dInfo("succcessed to read file %s", file);
+  dDebug("succcessed to read file %s", file);
   dmPrintDnodes(pMgmt);
 
 PRASE_DNODE_OVER:
@@ -200,7 +200,7 @@ int32_t dmWriteFile(SDnodeMgmt *pMgmt) {
   taosMemoryFree(content);
 
   char realfile[PATH_MAX];
-  snprintf(realfile, sizeof(realfile), "%s%smnode.json", pMgmt->path, TD_DIRSEP);
+  snprintf(realfile, sizeof(realfile), "%s%sdnode.json", pMgmt->path, TD_DIRSEP);
 
   if (taosRenameFile(file, realfile) != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);

source/dnode/mgmt/dm/src/dmInt.c

@@ -112,6 +112,16 @@ int32_t dmInit(SMgmtWrapper *pWrapper) {
     return -1;
   }
 
+  if (dndInitServer(pDnode) != 0) {
+    dError("failed to init trans server since %s", terrstr());
+    return -1;
+  }
+
+  if (dndInitClient(pDnode) != 0) {
+    dError("failed to init trans client since %s", terrstr());
+    return -1;
+  }
+
   pWrapper->pMgmt = pMgmt;
   dInfo("dnode-mgmt is initialized");
   return 0;
@@ -122,6 +132,7 @@ void dmCleanup(SMgmtWrapper *pWrapper) {
   if (pMgmt == NULL) return;
 
   dInfo("dnode-mgmt start to clean up");
+  SDnode *pDnode = pMgmt->pDnode;
   dmStopWorker(pMgmt);
 
   taosWLockLatch(&pMgmt->latch);
@@ -140,6 +151,9 @@ void dmCleanup(SMgmtWrapper *pWrapper) {
 
   taosMemoryFree(pMgmt);
   pWrapper->pMgmt = NULL;
+  dndCleanupServer(pDnode);
+  dndCleanupClient(pDnode);
+
   dInfo("dnode-mgmt is cleaned up");
 }
 

source/dnode/mgmt/dm/src/dmMsg.c

@@ -53,7 +53,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
   tSerializeSStatusReq(pHead, contLen, &req);
   taosArrayDestroy(req.pVloads);
 
-  SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)9527};
+  SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)0x9527};
   pMgmt->statusSent = 1;
 
   dTrace("send req:%s to mnode, app:%p", TMSG_INFO(rpcMsg.msgType), rpcMsg.ahandle);

source/dnode/mgmt/main/exe/dndMain.c

@@ -30,18 +30,13 @@ static struct {
 } global = {0};
 
 static void dndStopDnode(int signum, void *info, void *ctx) {
-  dInfo("signal:%d is received", signum);
+  dInfo("system signal:%d received", signum);
   SDnode *pDnode = atomic_val_compare_exchange_ptr(&global.pDnode, 0, global.pDnode);
   if (pDnode != NULL) {
     dndHandleEvent(pDnode, DND_EVENT_STOP);
   }
 }
 
-static void dndHandleChild(int signum, void *info, void *ctx) {
-  dInfo("signal:%d is received", signum);
-  dndHandleEvent(global.pDnode, DND_EVENT_CHILD);
-}
-
 static void dndSetSignalHandle() {
   taosSetSignal(SIGTERM, dndStopDnode);
   taosSetSignal(SIGHUP, dndStopDnode);
@@ -50,15 +45,10 @@ static void dndSetSignalHandle() {
   taosSetSignal(SIGBREAK, dndStopDnode);
 
   if (!tsMultiProcess) {
-    // Set the single process signal
-  } else if (global.ntype == DNODE) {
-    // Set the parent process signal
-    // When the child process exits, the parent process receives a signal
-    taosSetSignal(SIGCHLD, dndHandleChild);
+  } else if (global.ntype == DNODE || global.ntype == NODE_MAX) {
+    taosIgnSignal(SIGCHLD);
   } else {
-    // Set child process signal
-    // When the parent process exits, the child process will receive the SIGKILL signal
-    prctl(PR_SET_PDEATHSIG, SIGKILL);
+    taosKillChildOnParentStopped();
   }
 }
 
@@ -79,10 +69,14 @@ static int32_t dndParseArgs(int32_t argc, char const *argv[]) {
       tstrncpy(global.apolloUrl, argv[++i], PATH_MAX);
     } else if (strcmp(argv[i], "-e") == 0) {
       tstrncpy(global.envFile, argv[++i], PATH_MAX);
-    } else if (strcmp(argv[i], "-k") == 0) {
-      global.generateGrant = true;
     } else if (strcmp(argv[i], "-n") == 0) {
       global.ntype = atoi(argv[++i]);
+      if (global.ntype <= DNODE || global.ntype > NODE_MAX) {
+        printf("'-n' range is [1-5], default is 0\n");
+        return -1;
+      }
+    } else if (strcmp(argv[i], "-k") == 0) {
+      global.generateGrant = true;
     } else if (strcmp(argv[i], "-C") == 0) {
       global.dumpConfig = true;
     } else if (strcmp(argv[i], "-V") == 0) {
@@ -138,24 +132,24 @@ static int32_t dndInitLog() {
   return taosCreateLog(logName, 1, configDir, global.envFile, global.apolloUrl, global.pArgs, 0);
 }
 
-static void dndSetProcName(char **argv) {
-  if (global.ntype != DNODE) {
+static void dndSetProcInfo(int32_t argc, char **argv) {
+  taosSetProcPath(argc, argv);
+  if (global.ntype != DNODE && global.ntype != NODE_MAX) {
     const char *name = dndNodeProcStr(global.ntype);
-    prctl(PR_SET_NAME, name);
-    strcpy(argv[0], name);
+    taosSetProcName(argc, argv, name);
   }
 }
 
 static int32_t dndRunDnode() {
   if (dndInit() != 0) {
-    dError("failed to initialize environment since %s", terrstr());
+    dError("failed to init environment since %s", terrstr());
     return -1;
   }
 
   SDnodeOpt option = dndGetOpt();
   SDnode   *pDnode = dndCreate(&option);
   if (pDnode == NULL) {
-    dError("failed to to create dnode object since %s", terrstr());
+    dError("failed to to create dnode since %s", terrstr());
     return -1;
   } else {
     global.pDnode = pDnode;
@@ -212,6 +206,6 @@ int main(int argc, char const *argv[]) {
     return 0;
   }
 
-  dndSetProcName((char **)argv);
+  dndSetProcInfo(argc, (char **)argv);
   return dndRunDnode();
 }

source/dnode/mgmt/main/inc/dnd.h

@@ -95,13 +95,14 @@ typedef struct SMgmtWrapper {
   bool        deployed;
   bool        required;
   EProcType   procType;
+  int32_t     procId;
   SProcObj   *pProc;
   SShm        shm;
   void       *pMgmt;
   SDnode     *pDnode;
-  NodeMsgFp   msgFps[TDMT_MAX];
-  int8_t      msgVgIds[TDMT_MAX];  // Handle the case where the same message type is distributed to qnode or vnode
   SMgmtFp     fp;
+  int8_t      msgVgIds[TDMT_MAX];  // Handle the case where the same message type is distributed to qnode or vnode
+  NodeMsgFp   msgFps[TDMT_MAX];
 } SMgmtWrapper;
 
 typedef struct {
@@ -128,26 +129,31 @@ typedef struct SDnode {
   EDndStatus   status;
   EDndEvent    event;
   SStartupReq  startup;
-  TdFilePtr    runtimeFile;
+  TdFilePtr    lockfile;
   STransMgmt   trans;
   SMgmtWrapper wrappers[NODE_MAX];
 } SDnode;
 
 const char *dndNodeLogStr(ENodeType ntype);
 const char *dndNodeProcStr(ENodeType ntype);
+const char *dndEventStr(EDndEvent ev);
 EDndStatus  dndGetStatus(SDnode *pDnode);
 void        dndSetStatus(SDnode *pDnode, EDndStatus stat);
 void        dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId);
 void        dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc);
 void        dndSendMonitorReport(SDnode *pDnode);
 
+int32_t dndInitServer(SDnode *pDnode);
+void    dndCleanupServer(SDnode *pDnode);
+int32_t dndInitClient(SDnode *pDnode);
+void    dndCleanupClient(SDnode *pDnode);
+int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg);
 int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
 int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pMsg);
 void    dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp);
 void    dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
 SMsgCb  dndCreateMsgcb(SMgmtWrapper *pWrapper);
 
-int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg);
 int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed);
 int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed);
 

source/dnode/mgmt/main/inc/dndInt.h

@@ -50,17 +50,13 @@ void    dndClose(SDnode *pDnode);
 void    dndHandleEvent(SDnode *pDnode, EDndEvent event);
 
 // dndTransport.c
-int32_t dndInitServer(SDnode *pDnode);
-void    dndCleanupServer(SDnode *pDnode);
-int32_t dndInitClient(SDnode *pDnode);
-void    dndCleanupClient(SDnode *pDnode);
 int32_t dndInitMsgHandle(SDnode *pDnode);
 void    dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp);
 
 // dndFile.c
-int32_t dndOpenRuntimeFile(SDnode *pDnode);
-int32_t dndWriteRuntimeFile(SDnode *pDnode);
-void    dndCloseRuntimeFile(SDnode *pDnode);
+TdFilePtr dndCheckRunning(const char *dataDir);
+int32_t   dndReadShmFile(SDnode *pDnode);
+int32_t   dndWriteShmFile(SDnode *pDnode);
 
 #ifdef __cplusplus
 }

source/dnode/mgmt/main/src/dndEnv.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dndInt.h"
+#include "wal.h"
+
+static int8_t once = DND_ENV_INIT;
+
+int32_t dndInit() {
+  dDebug("start to init dnode env");
+  if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) {
+    terrno = TSDB_CODE_REPEAT_INIT;
+    dError("failed to init dnode env since %s", terrstr());
+    return -1;
+  }
+
+  taosIgnSIGPIPE();
+  taosBlockSIGPIPE();
+  taosResolveCRC();
+
+  SMonCfg monCfg = {0};
+  monCfg.maxLogs = tsMonitorMaxLogs;
+  monCfg.port = tsMonitorPort;
+  monCfg.server = tsMonitorFqdn;
+  monCfg.comp = tsMonitorComp;
+  if (monInit(&monCfg) != 0) {
+    dError("failed to init monitor since %s", terrstr());
+    return -1;
+  }
+
+  dInfo("dnode env is initialized");
+  return 0;
+}
+
+void dndCleanup() {
+  dDebug("start to cleanup dnode env");
+  if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) {
+    dError("dnode env is already cleaned up");
+    return;
+  }
+
+  monCleanup();
+  walCleanUp();
+  taosStopCacheRefreshWorker();
+  dInfo("dnode env is cleaned up");
+}
+
+void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId) {
+  pWrapper->msgFps[TMSG_INDEX(msgType)] = nodeMsgFp;
+  pWrapper->msgVgIds[TMSG_INDEX(msgType)] = vgId;
+}
+
+EDndStatus dndGetStatus(SDnode *pDnode) { return pDnode->status; }
+
+void dndSetStatus(SDnode *pDnode, EDndStatus status) {
+  if (pDnode->status != status) {
+    dDebug("dnode status set from %s to %s", dndStatStr(pDnode->status), dndStatStr(status));
+    pDnode->status = status;
+  }
+}
+
+void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) {
+  SStartupReq *pStartup = &pDnode->startup;
+  tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN);
+  tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN);
+  pStartup->finished = 0;
+}
+
+void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) {
+  memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq));
+  pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING);
+}
+
+void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) {
+  dDebug("startup req is received");
+  SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq));
+  dndGetStartup(pDnode, pStartup);
+
+  dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished);
+  SRpcMsg rpcRsp = {
+      .handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq), .ahandle = pReq->ahandle};
+  rpcSendResponse(&rpcRsp);
+}

source/dnode/mgmt/main/src/dndExec.c

@@ -16,35 +16,30 @@
 #define _DEFAULT_SOURCE
 #include "dndInt.h"
 
-static void dndResetLog(SMgmtWrapper *pMgmt) {
-  char logname[24] = {0};
-  snprintf(logname, sizeof(logname), "%slog", pMgmt->name);
-
-  dInfo("node:%s, reset log to %s in child process", pMgmt->name, logname);
-  taosCloseLog();
-  taosInitLog(logname, 1);
-}
-
 static bool dndRequireNode(SMgmtWrapper *pWrapper) {
   bool    required = false;
-  int32_t code =(*pWrapper->fp.requiredFp)(pWrapper, &required);
+  int32_t code = (*pWrapper->fp.requiredFp)(pWrapper, &required);
   if (!required) {
-    dDebug("node:%s, no need to start", pWrapper->name);
+    dDebug("node:%s, does not require startup", pWrapper->name);
   } else {
-    dDebug("node:%s, need to start", pWrapper->name);
+    dDebug("node:%s, needs to be started", pWrapper->name);
   }
   return required;
 }
 
 int32_t dndOpenNode(SMgmtWrapper *pWrapper) {
-  int32_t code = (*pWrapper->fp.openFp)(pWrapper);
-  if (code != 0) {
+  if (taosMkDir(pWrapper->path) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("node:%s, failed to create dir:%s since %s", pWrapper->name, pWrapper->path, terrstr());
+    return -1;
+  }
+
+  if ((*pWrapper->fp.openFp)(pWrapper) != 0) {
     dError("node:%s, failed to open since %s", pWrapper->name, terrstr());
     return -1;
-  } else {
-    dDebug("node:%s, has been opened", pWrapper->name);
   }
 
+  dDebug("node:%s, has been opened", pWrapper->name);
   pWrapper->deployed = true;
   return 0;
 }
@@ -70,54 +65,6 @@ void dndCloseNode(SMgmtWrapper *pWrapper) {
   dDebug("node:%s, has been closed", pWrapper->name);
 }
 
-static int32_t dndRunInSingleProcess(SDnode *pDnode) {
-  dInfo("dnode run in single process mode");
-
-  for (ENodeType n = 0; n < NODE_MAX; ++n) {
-    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
-    pWrapper->required = dndRequireNode(pWrapper);
-    if (!pWrapper->required) continue;
-    SMsgCb msgCb = dndCreateMsgcb(pWrapper);
-    tmsgSetDefaultMsgCb(&msgCb);
-
-    if (taosMkDir(pWrapper->path) != 0) {
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      dError("failed to create dir:%s since %s", pWrapper->path, terrstr());
-      return -1;
-    }
-
-    dInfo("node:%s, will start in single process", pWrapper->name);
-    pWrapper->procType = PROC_SINGLE;
-    if (dndOpenNode(pWrapper) != 0) {
-      dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
-      return -1;
-    }
-  }
-
-  dndSetStatus(pDnode, DND_STAT_RUNNING);
-
-  for (ENodeType n = 0; n < NODE_MAX; ++n) {
-    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
-    if (!pWrapper->required) continue;
-    if (pWrapper->fp.startFp == NULL) continue;
-    if ((*pWrapper->fp.startFp)(pWrapper) != 0) {
-      dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
-      return -1;
-    }
-  }
-
-  return 0;
-}
-
-static void dndClearNodesExecpt(SDnode *pDnode, ENodeType except) {
-  // dndCleanupServer(pDnode);
-  for (ENodeType n = 0; n < NODE_MAX; ++n) {
-    if (except == n) continue;
-    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
-    pWrapper->required = false;
-  }
-}
-
 static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen,
                                  ProcFuncType ftype) {
   SRpcMsg *pRpc = &pMsg->rpcMsg;
@@ -163,116 +110,234 @@ static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t
   taosMemoryFree(pMsg);
 }
 
-static int32_t dndRunInMultiProcess(SDnode *pDnode) {
-  dInfo("dnode run in multi process mode");
-
-  for (ENodeType n = 0; n < NODE_MAX; ++n) {
-    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
-    pWrapper->required = dndRequireNode(pWrapper);
-    if (!pWrapper->required) continue;
-
-    SMsgCb msgCb = dndCreateMsgcb(pWrapper);
-    tmsgSetDefaultMsgCb(&msgCb);
-
-    if (taosMkDir(pWrapper->path) != 0) {
+static int32_t dndNewProc(SMgmtWrapper *pWrapper, ENodeType n) {
+  char  tstr[8] = {0};
+  char *args[6] = {0};
+  snprintf(tstr, sizeof(tstr), "%d", n);
+  args[1] = "-c";
+  args[2] = configDir;
+  args[3] = "-n";
+  args[4] = tstr;
+  args[5] = NULL;
+
+  int32_t pid = taosNewProc(args);
+  if (pid <= 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
-      dError("failed to create dir:%s since %s", pWrapper->path, terrstr());
+    dError("node:%s, failed to exec in new process since %s", pWrapper->name, terrstr());
     return -1;
   }
 
-    if (n == DNODE) {
-      dInfo("node:%s, will start in parent process", pWrapper->name);
-      pWrapper->procType = PROC_SINGLE;
-      if (dndOpenNode(pWrapper) != 0) {
-        dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
-        return -1;
-      }
-      continue;
-    }
+  pWrapper->procId = pid;
+  dInfo("node:%s, continue running in new process:%d", pWrapper->name, pid);
+  return 0;
+}
 
-    SProcCfg  cfg = {.childQueueSize = 1024 * 1024 * 2,  // size will be a configuration item
-                     .childConsumeFp = (ProcConsumeFp)dndConsumeChildQueue,
+static SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper) {
+  SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)dndConsumeChildQueue,
                   .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem,
                   .childFreeHeadFp = (ProcFreeFp)taosFreeQitem,
                   .childMallocBodyFp = (ProcMallocFp)rpcMallocCont,
                   .childFreeBodyFp = (ProcFreeFp)rpcFreeCont,
-                     .parentQueueSize = 1024 * 1024 * 2,  // size will be a configuration item
                   .parentConsumeFp = (ProcConsumeFp)dndConsumeParentQueue,
-                     .parentdMallocHeadFp = (ProcMallocFp)taosMemoryMalloc,
+                  .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc,
                   .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree,
                   .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont,
                   .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont,
+                  .shm = pWrapper->shm,
                   .pParent = pWrapper,
                   .name = pWrapper->name};
-    SProcObj *pProc = taosProcInit(&cfg);
-    if (pProc == NULL) {
-      dError("node:%s, failed to fork since %s", pWrapper->name, terrstr());
+  return cfg;
+}
+
+static int32_t dndRunInSingleProcess(SDnode *pDnode) {
+  dInfo("dnode run in single process");
+
+  for (ENodeType n = DNODE; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+    pWrapper->required = dndRequireNode(pWrapper);
+    if (!pWrapper->required) continue;
+
+    if (dndOpenNode(pWrapper) != 0) {
+      dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
       return -1;
     }
+  }
 
-    pWrapper->pProc = pProc;
+  dndSetStatus(pDnode, DND_STAT_RUNNING);
 
-    if (taosProcIsChild(pProc)) {
-      dInfo("node:%s, will start in child process", pWrapper->name);
-      pWrapper->procType = PROC_CHILD;
-      dndResetLog(pWrapper);
+  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+    if (!pWrapper->required) continue;
+    if (pWrapper->fp.startFp == NULL) continue;
+    if ((*pWrapper->fp.startFp)(pWrapper) != 0) {
+      dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
+      return -1;
+    }
+  }
+
+  dInfo("TDengine initialized successfully");
+  dndReportStartup(pDnode, "TDengine", "initialized successfully");
+  while (1) {
+    if (pDnode->event == DND_EVENT_STOP) {
+      dInfo("dnode is about to stop");
+      break;
+    }
+    taosMsleep(100);
+  }
 
-      dInfo("node:%s, clean up resources inherited from parent", pWrapper->name);
-      dndClearNodesExecpt(pDnode, n);
+  return 0;
+}
 
-      dInfo("node:%s, will be initialized in child process", pWrapper->name);
-      if (dndOpenNode(pWrapper) != 0) {
-        dInfo("node:%s, failed to init in child process since %s", pWrapper->name, terrstr());
+static int32_t dndRunInParentProcess(SDnode *pDnode) {
+  dInfo("dnode run in parent process");
+  SMgmtWrapper *pDWrapper = &pDnode->wrappers[DNODE];
+  if (dndOpenNode(pDWrapper) != 0) {
+    dError("node:%s, failed to start since %s", pDWrapper->name, terrstr());
     return -1;
   }
 
-      if (taosProcRun(pProc) != 0) {
-        dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr());
+  for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+    pWrapper->required = dndRequireNode(pWrapper);
+    if (!pWrapper->required) continue;
+
+    int32_t shmsize = 1024 * 1024 * 2;  // size will be a configuration item
+    if (taosCreateShm(&pWrapper->shm, shmsize) != 0) {
+      terrno = TAOS_SYSTEM_ERROR(terrno);
+      dError("node:%s, failed to create shm size:%d since %s", pWrapper->name, shmsize, terrstr());
       return -1;
     }
-      break;
-    } else {
-      dInfo("node:%s, will not start in parent process, child pid:%d", pWrapper->name, taosProcChildId(pProc));
+    dInfo("node:%s, shm:%d is created, size:%d", pWrapper->name, pWrapper->shm.id, shmsize);
+
+    SProcCfg cfg = dndGenProcCfg(pWrapper);
+    cfg.isChild = false;
     pWrapper->procType = PROC_PARENT;
-      if (taosProcRun(pProc) != 0) {
-        dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr());
+    pWrapper->pProc = taosProcInit(&cfg);
+    if (pWrapper->pProc == NULL) {
+      dError("node:%s, failed to create proc since %s", pWrapper->name, terrstr());
+      return -1;
+    }
+  }
+
+  if (dndWriteShmFile(pDnode) != 0) {
+    dError("failed to write runtime file since %s", terrstr());
+    return -1;
+  }
+
+  for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+    if (!pWrapper->required) continue;
+
+    if (pDnode->ntype == NODE_MAX) {
+      dInfo("node:%s, should be started manually", pWrapper->name);
+    } else {
+      if (dndNewProc(pWrapper, n) != 0) {
         return -1;
       }
     }
+
+    if (taosProcRun(pWrapper->pProc) != 0) {
+      dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr());
+      return -1;
+    }
   }
 
   dndSetStatus(pDnode, DND_STAT_RUNNING);
 
-  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+  if ((*pDWrapper->fp.startFp)(pDWrapper) != 0) {
+    dError("node:%s, failed to start since %s", pDWrapper->name, terrstr());
+    return -1;
+  }
+
+  dInfo("TDengine initialized successfully");
+  dndReportStartup(pDnode, "TDengine", "initialized successfully");
+
+  while (1) {
+    if (pDnode->event == DND_EVENT_STOP) {
+      dInfo("dnode is about to stop");
+      for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) {
         SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
         if (!pWrapper->required) continue;
-    if (pWrapper->fp.startFp == NULL) continue;
-    if (pWrapper->procType == PROC_PARENT && n != DNODE) continue;
-    if (pWrapper->procType == PROC_CHILD && n == DNODE) continue;
-    if ((*pWrapper->fp.startFp)(pWrapper) != 0) {
-      dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
-      return -1;
+        if (pDnode->ntype == NODE_MAX) continue;
+
+        if (pWrapper->procId > 0 && taosProcExist(pWrapper->procId)) {
+          dInfo("node:%s, send kill signal to the child process:%d", pWrapper->name, pWrapper->procId);
+          taosKillProc(pWrapper->procId);
         }
       }
 
+      for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) {
+        SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+        if (!pWrapper->required) continue;
+        if (pDnode->ntype == NODE_MAX) continue;
+
+        if (pWrapper->procId > 0 && taosProcExist(pWrapper->procId)) {
+          dInfo("node:%s, wait for child process:%d to stop", pWrapper->name, pWrapper->procId);
+          taosWaitProc(pWrapper->procId);
+          dInfo("node:%s, child process:%d is stopped", pWrapper->name, pWrapper->procId);
+        }
+      }
+      break;
+    } else {
+      for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) {
+        SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+        if (!pWrapper->required) continue;
+        if (pDnode->ntype == NODE_MAX) continue;
+
+        if (pWrapper->procId <= 0 || !taosProcExist(pWrapper->procId)) {
+          dInfo("node:%s, process:%d is killed and needs to be restarted", pWrapper->name, pWrapper->procId);
+          dndNewProc(pWrapper, n);
+        }
+      }
+    }
+
+    taosMsleep(100);
+  }
+
   return 0;
 }
 
-int32_t dndRun(SDnode *pDnode) {
-  if (!tsMultiProcess) {
-    if (dndRunInSingleProcess(pDnode) != 0) {
-      dError("failed to run dnode in single process mode since %s", terrstr());
+static int32_t dndRunInChildProcess(SDnode *pDnode) {
+  SMgmtWrapper *pWrapper = &pDnode->wrappers[pDnode->ntype];
+  dInfo("%s run in child process", pWrapper->name);
+
+  pWrapper->required = dndRequireNode(pWrapper);
+  if (!pWrapper->required) {
+    dError("%s does not require startup", pWrapper->name);
     return -1;
   }
-  } else {
-    if (dndRunInMultiProcess(pDnode) != 0) {
-      dError("failed to run dnode in multi process mode since %s", terrstr());
+
+  SMsgCb msgCb = dndCreateMsgcb(pWrapper);
+  tmsgSetDefaultMsgCb(&msgCb);
+  pWrapper->procType = PROC_CHILD;
+
+  if (dndOpenNode(pWrapper) != 0) {
+    dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
     return -1;
   }
+
+  SProcCfg cfg = dndGenProcCfg(pWrapper);
+  cfg.isChild = true;
+  pWrapper->pProc = taosProcInit(&cfg);
+  if (pWrapper->pProc == NULL) {
+    dError("node:%s, failed to create proc since %s", pWrapper->name, terrstr());
+    return -1;
   }
 
-  dndReportStartup(pDnode, "TDengine", "initialized successfully");
+  if (pWrapper->fp.startFp != NULL) {
+    if ((*pWrapper->fp.startFp)(pWrapper) != 0) {
+      dError("node:%s, failed to start since %s", pWrapper->name, terrstr());
+      return -1;
+    }
+  }
 
+  if (taosProcRun(pWrapper->pProc) != 0) {
+    dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr());
+    return -1;
+  }
+
+  dInfo("TDengine initialized successfully");
+  dndReportStartup(pDnode, "TDengine", "initialized successfully");
   while (1) {
     if (pDnode->event == DND_EVENT_STOP) {
       dInfo("dnode is about to stop");
@@ -280,6 +345,16 @@ int32_t dndRun(SDnode *pDnode) {
     }
     taosMsleep(100);
   }
+}
+
+int32_t dndRun(SDnode *pDnode) {
+  if (!tsMultiProcess) {
+    return dndRunInSingleProcess(pDnode);
+  } else if (pDnode->ntype == DNODE || pDnode->ntype == NODE_MAX) {
+    return dndRunInParentProcess(pDnode);
+  } else {
+    return dndRunInChildProcess(pDnode);
+  }
 
   return 0;
 }

source/dnode/mgmt/main/src/dndFile.c

@@ -117,7 +117,30 @@ _OVER:
   return code;
 }
 
-int32_t dndOpenRuntimeFile(SDnode *pDnode) {
+TdFilePtr dndCheckRunning(const char *dataDir) {
+  char filepath[PATH_MAX] = {0};
+  snprintf(filepath, sizeof(filepath), "%s%s.running", dataDir, TD_DIRSEP);
+
+  TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to lock file:%s since %s", filepath, terrstr());
+    return NULL;
+  }
+
+  int32_t ret = taosLockFile(pFile);
+  if (ret != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to lock file:%s since %s", filepath, terrstr());
+    taosCloseFile(&pFile);
+    return NULL;
+  }
+
+  dDebug("file:%s is locked", filepath);
+  return pFile;
+}
+
+int32_t dndReadShmFile(SDnode *pDnode) {
   int32_t   code = -1;
   char      itemName[24] = {0};
   char      content[MAXLEN + 1] = {0};
@@ -125,17 +148,11 @@ int32_t dndOpenRuntimeFile(SDnode *pDnode) {
   cJSON    *root = NULL;
   TdFilePtr pFile = NULL;
 
-  snprintf(file, sizeof(file), "%s%s.running", pDnode->dataDir, TD_DIRSEP);
-  pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  snprintf(file, sizeof(file), "%s%s.shmfile", pDnode->dataDir, TD_DIRSEP);
+  pFile = taosOpenFile(file, TD_FILE_READ);
   if (pFile == NULL) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    dError("failed to open file:%s since %s", file, terrstr());
-    goto _OVER;
-  }
-
-  if (taosLockFile(pFile) != 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    dError("failed to lock file:%s since %s", file, terrstr());
+    dDebug("file %s not exist", file);
+    code = 0;
     goto _OVER;
   }
 
@@ -162,10 +179,10 @@ int32_t dndOpenRuntimeFile(SDnode *pDnode) {
     }
   }
 
-  if (tsMultiProcess || pDnode->ntype == DNODE) {
+  if (!tsMultiProcess || pDnode->ntype == DNODE || pDnode->ntype == NODE_MAX) {
     for (ENodeType ntype = DNODE; ntype < NODE_MAX; ++ntype) {
-      SMgmtWrapper *pWrapper = &pDnode->wrappers[pDnode->ntype];
-      if (pWrapper->shm.id > 0) {
+      SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
+      if (pWrapper->shm.id >= 0) {
         dDebug("shmid:%d, is closed, size:%d", pWrapper->shm.id, pWrapper->shm.size);
         taosDropShm(&pWrapper->shm);
       }
@@ -174,27 +191,23 @@ int32_t dndOpenRuntimeFile(SDnode *pDnode) {
     SMgmtWrapper *pWrapper = &pDnode->wrappers[pDnode->ntype];
     if (taosAttachShm(&pWrapper->shm) != 0) {
       terrno = TAOS_SYSTEM_ERROR(errno);
-      dError("shmid:%d, failed to attach since %s", pWrapper->shm.id, terrstr());
+      dError("shmid:%d, failed to attach shm since %s", pWrapper->shm.id, terrstr());
       goto _OVER;
     }
-    dDebug("shmid:%d, is attached, size:%d", pWrapper->shm.id, pWrapper->shm.size);
+    dInfo("node:%s, shmid:%d is attached, size:%d", pWrapper->name, pWrapper->shm.id, pWrapper->shm.size);
   }
 
-  dDebug("successed to open %s", file);
+  dDebug("successed to load %s", file);
   code = 0;
 
 _OVER:
   if (root != NULL) cJSON_Delete(root);
-  if (code != 0) {
   if (pFile != NULL) taosCloseFile(&pFile);
-  } else {
-    pDnode->runtimeFile = pFile;
-  }
 
   return code;
 }
 
-int32_t dndWriteRuntimeFile(SDnode *pDnode) {
+int32_t dndWriteShmFile(SDnode *pDnode) {
   int32_t   code = -1;
   int32_t   len = 0;
   char      content[MAXLEN + 1] = {0};
@@ -202,8 +215,8 @@ int32_t dndWriteRuntimeFile(SDnode *pDnode) {
   char      realfile[PATH_MAX] = {0};
   TdFilePtr pFile = NULL;
 
-  snprintf(file, sizeof(file), "%s%s.running.bak", pDnode->dataDir, TD_DIRSEP);
-  snprintf(realfile, sizeof(realfile), "%s%s.running", pDnode->dataDir, TD_DIRSEP);
+  snprintf(file, sizeof(file), "%s%s.shmfile.bak", pDnode->dataDir, TD_DIRSEP);
+  snprintf(realfile, sizeof(realfile), "%s%s.shmfile", pDnode->dataDir, TD_DIRSEP);
 
   pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
@@ -214,12 +227,12 @@ int32_t dndWriteRuntimeFile(SDnode *pDnode) {
 
   len += snprintf(content + len, MAXLEN - len, "{\n");
   for (ENodeType ntype = DNODE + 1; ntype < NODE_MAX; ++ntype) {
-    SMgmtWrapper *pWrapper = &pDnode->wrappers[pDnode->ntype];
-    len += snprintf(content + len, MAXLEN - len, "  \"%s_shmid\": %d,\n", dndNodeProcStr(ntype), pWrapper->shm.id);
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
+    len += snprintf(content + len, MAXLEN - len, "  \"%s_shmid\":%d,\n", dndNodeProcStr(ntype), pWrapper->shm.id);
     if (ntype == NODE_MAX - 1) {
-      len += snprintf(content + len, MAXLEN - len, "  \"%s_shmsize\": %d\n", dndNodeProcStr(ntype), pWrapper->shm.size);
+      len += snprintf(content + len, MAXLEN - len, "  \"%s_shmsize\":%d\n", dndNodeProcStr(ntype), pWrapper->shm.size);
     } else {
-      len += snprintf(content + len, MAXLEN - len, "  \"%s_shmsize\": %d,\n", dndNodeProcStr(ntype), pWrapper->shm.size);
+      len += snprintf(content + len, MAXLEN - len, "  \"%s_shmsize\":%d,\n", dndNodeProcStr(ntype), pWrapper->shm.size);
     }
   }
   len += snprintf(content + len, MAXLEN - len, "}\n");
@@ -244,7 +257,7 @@ int32_t dndWriteRuntimeFile(SDnode *pDnode) {
     return -1;
   }
 
-  dDebug("successed to write %s", realfile);
+  dInfo("successed to write %s", realfile);
   code = 0;
 
 _OVER:
@@ -254,11 +267,3 @@ _OVER:
 
   return code;
 }
-
-void dndCloseRuntimeFile(SDnode *pDnode) {
-  if (pDnode->runtimeFile) {
-    taosUnLockFile(pDnode->runtimeFile);
-    taosCloseFile(&pDnode->runtimeFile);
-    pDnode->runtimeFile = NULL;
-  }
-}
\ No newline at end of file

source/dnode/mgmt/main/src/dndInt.c

@@ -15,82 +15,186 @@
 
 #define _DEFAULT_SOURCE
 #include "dndInt.h"
-#include "wal.h"
 
-static int8_t once = DND_ENV_INIT;
+static int32_t dndInitVars(SDnode *pDnode, const SDnodeOpt *pOption) {
+  pDnode->numOfSupportVnodes = pOption->numOfSupportVnodes;
+  pDnode->serverPort = pOption->serverPort;
+  pDnode->dataDir = strdup(pOption->dataDir);
+  pDnode->localEp = strdup(pOption->localEp);
+  pDnode->localFqdn = strdup(pOption->localFqdn);
+  pDnode->firstEp = strdup(pOption->firstEp);
+  pDnode->secondEp = strdup(pOption->secondEp);
+  pDnode->disks = pOption->disks;
+  pDnode->numOfDisks = pOption->numOfDisks;
+  pDnode->ntype = pOption->ntype;
+  pDnode->rebootTime = taosGetTimestampMs();
 
-int32_t dndInit() {
-  dDebug("start to init dnode env");
-  if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) {
-    terrno = TSDB_CODE_REPEAT_INIT;
-    dError("failed to init dnode env since %s", terrstr());
+  if (pDnode->dataDir == NULL || pDnode->localEp == NULL || pDnode->localFqdn == NULL || pDnode->firstEp == NULL ||
+      pDnode->secondEp == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
 
-  taosIgnSIGPIPE();
-  taosBlockSIGPIPE();
-  taosResolveCRC();
-
-  SMonCfg monCfg = {0};
-  monCfg.maxLogs = tsMonitorMaxLogs;
-  monCfg.port = tsMonitorPort;
-  monCfg.server = tsMonitorFqdn;
-  monCfg.comp = tsMonitorComp;
-  if (monInit(&monCfg) != 0) {
-    dError("failed to init monitor since %s", terrstr());
+  if (!tsMultiProcess || pDnode->ntype == DNODE || pDnode->ntype == NODE_MAX) {
+    pDnode->lockfile = dndCheckRunning(pDnode->dataDir);
+    if (pDnode->lockfile == NULL) {
       return -1;
     }
+  }
 
-  dDebug("dnode env is initialized");
   return 0;
 }
 
-void dndCleanup() {
-  dDebug("start to cleanup dnode env");
-  if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) {
-    dError("dnode env is already cleaned up");
+static void dndClearVars(SDnode *pDnode) {
+  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pMgmt = &pDnode->wrappers[n];
+    taosMemoryFreeClear(pMgmt->path);
+  }
+  if (pDnode->lockfile != NULL) {
+    taosUnLockFile(pDnode->lockfile);
+    taosCloseFile(&pDnode->lockfile);
+    pDnode->lockfile = NULL;
+  }
+  taosMemoryFreeClear(pDnode->localEp);
+  taosMemoryFreeClear(pDnode->localFqdn);
+  taosMemoryFreeClear(pDnode->firstEp);
+  taosMemoryFreeClear(pDnode->secondEp);
+  taosMemoryFreeClear(pDnode->dataDir);
+  taosMemoryFree(pDnode);
+  dDebug("dnode memory is cleared, data:%p", pDnode);
+}
+
+SDnode *dndCreate(const SDnodeOpt *pOption) {
+  dDebug("start to create dnode object");
+  int32_t code = -1;
+  char    path[PATH_MAX] = {0};
+  SDnode *pDnode = NULL;
+
+  pDnode = taosMemoryCalloc(1, sizeof(SDnode));
+  if (pDnode == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto _OVER;
+  }
+
+  if (dndInitVars(pDnode, pOption) != 0) {
+    dError("failed to init variables since %s", terrstr());
+    goto _OVER;
+  }
+
+  dndSetStatus(pDnode, DND_STAT_INIT);
+  dmGetMgmtFp(&pDnode->wrappers[DNODE]);
+  mmGetMgmtFp(&pDnode->wrappers[MNODE]);
+  vmGetMgmtFp(&pDnode->wrappers[VNODES]);
+  qmGetMgmtFp(&pDnode->wrappers[QNODE]);
+  smGetMgmtFp(&pDnode->wrappers[SNODE]);
+  bmGetMgmtFp(&pDnode->wrappers[BNODE]);
+
+  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+    snprintf(path, sizeof(path), "%s%s%s", pDnode->dataDir, TD_DIRSEP, pWrapper->name);
+    pWrapper->path = strdup(path);
+    pWrapper->shm.id = -1;
+    pWrapper->pDnode = pDnode;
+    if (pWrapper->path == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      goto _OVER;
+    }
+
+    pWrapper->procType = PROC_SINGLE;
+    taosInitRWLatch(&pWrapper->latch);
+  }
+
+  if (dndInitMsgHandle(pDnode) != 0) {
+    dError("failed to msg handles since %s", terrstr());
+    goto _OVER;
+  }
+
+  if (dndReadShmFile(pDnode) != 0) {
+    dError("failed to read shm file since %s", terrstr());
+    goto _OVER;
+  }
+
+  SMsgCb msgCb = dndCreateMsgcb(&pDnode->wrappers[0]);
+  tmsgSetDefaultMsgCb(&msgCb);
+
+  dInfo("dnode is created, data:%p", pDnode);
+  code = 0;
+
+_OVER:
+  if (code != 0 && pDnode) {
+    dndClearVars(pDnode);
+    pDnode = NULL;
+    dError("failed to create dnode since %s", terrstr());
+  }
+
+  return pDnode;
+}
+
+void dndClose(SDnode *pDnode) {
+  if (pDnode == NULL) return;
+
+  if (dndGetStatus(pDnode) == DND_STAT_STOPPED) {
+    dError("dnode is shutting down, data:%p", pDnode);
     return;
   }
 
-  monCleanup();
-  walCleanUp();
-  taosStopCacheRefreshWorker();
-  dDebug("dnode env is cleaned up");
+  dInfo("start to close dnode, data:%p", pDnode);
+  dndSetStatus(pDnode, DND_STAT_STOPPED);
+
+  for (ENodeType n = 0; n < NODE_MAX; ++n) {
+    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
+    dndCloseNode(pWrapper);
+  }
+
+  dndClearVars(pDnode);
+  dInfo("dnode is closed, data:%p", pDnode);
 }
 
-void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId) {
-  pWrapper->msgFps[TMSG_INDEX(msgType)] = nodeMsgFp;
-  pWrapper->msgVgIds[TMSG_INDEX(msgType)] = vgId;
+void dndHandleEvent(SDnode *pDnode, EDndEvent event) {
+  dInfo("dnode receive %s event, data:%p", dndEventStr(event), pDnode);
+  if (event == DND_EVENT_STOP) {
+    pDnode->event = event;
+  }
 }
 
-EDndStatus dndGetStatus(SDnode *pDnode) { return pDnode->status; }
+SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType ntype) {
+  SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
+  SMgmtWrapper *pRetWrapper = pWrapper;
 
-void dndSetStatus(SDnode *pDnode, EDndStatus status) {
-  if (pDnode->status != status) {
-    dDebug("dnode status set from %s to %s", dndStatStr(pDnode->status), dndStatStr(status));
-    pDnode->status = status;
+  taosRLockLatch(&pWrapper->latch);
+  if (pWrapper->deployed) {
+    int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1);
+    dTrace("node:%s, is acquired, refCount:%d", pWrapper->name, refCount);
+  } else {
+    terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
+    pRetWrapper = NULL;
   }
-}
+  taosRUnLockLatch(&pWrapper->latch);
 
-void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) {
-  SStartupReq *pStartup = &pDnode->startup;
-  tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN);
-  tstrncpy(pStartup->desc, pDesc, TSDB_STEP_DESC_LEN);
-  pStartup->finished = 0;
+  return pRetWrapper;
 }
 
-void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) {
-  memcpy(pStartup, &pDnode->startup, sizeof(SStartupReq));
-  pStartup->finished = (dndGetStatus(pDnode) == DND_STAT_RUNNING);
+int32_t dndMarkWrapper(SMgmtWrapper *pWrapper) {
+  int32_t code = 0;
+
+  taosRLockLatch(&pWrapper->latch);
+  if (pWrapper->deployed || (pWrapper->procType == PROC_PARENT && pWrapper->required)) {
+    int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1);
+    dTrace("node:%s, is marked, refCount:%d", pWrapper->name, refCount);
+  } else {
+    terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
+    code = -1;
+  }
+  taosRUnLockLatch(&pWrapper->latch);
+
+  return code;
 }
 
-void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) {
-  dDebug("startup req is received");
-  SStartupReq *pStartup = rpcMallocCont(sizeof(SStartupReq));
-  dndGetStartup(pDnode, pStartup);
+void dndReleaseWrapper(SMgmtWrapper *pWrapper) {
+  if (pWrapper == NULL) return;
 
-  dDebug("startup req is sent, step:%s desc:%s finished:%d", pStartup->name, pStartup->desc, pStartup->finished);
-  SRpcMsg rpcRsp = {
-      .handle = pReq->handle, .pCont = pStartup, .contLen = sizeof(SStartupReq), .ahandle = pReq->ahandle};
-  rpcSendResponse(&rpcRsp);
+  taosRLockLatch(&pWrapper->latch);
+  int32_t refCount = atomic_sub_fetch_32(&pWrapper->refCount, 1);
+  taosRUnLockLatch(&pWrapper->latch);
+  dTrace("node:%s, is released, refCount:%d", pWrapper->name, refCount);
 }
\ No newline at end of file

source/dnode/mgmt/main/src/dndMsg.c

@@ -66,7 +66,7 @@ void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) {
     dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user);
     code = (*msgFp)(pWrapper, pMsg);
   } else if (pWrapper->procType == PROC_PARENT) {
-    dTrace("msg:%p, is created and will put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle,
+    dTrace("msg:%p, is created and put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle,
            pRpc->ahandle, pMsg->user);
     code = taosProcPutToChildQ(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, PROC_REQ);
   } else {

source/dnode/mgmt/main/src/dndObj.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "dndInt.h"
-
-static int32_t dndInitVars(SDnode *pDnode, const SDnodeOpt *pOption) {
-  pDnode->numOfSupportVnodes = pOption->numOfSupportVnodes;
-  pDnode->serverPort = pOption->serverPort;
-  pDnode->dataDir = strdup(pOption->dataDir);
-  pDnode->localEp = strdup(pOption->localEp);
-  pDnode->localFqdn = strdup(pOption->localFqdn);
-  pDnode->firstEp = strdup(pOption->firstEp);
-  pDnode->secondEp = strdup(pOption->secondEp);
-  pDnode->disks = pOption->disks;
-  pDnode->numOfDisks = pOption->numOfDisks;
-  pDnode->ntype = pOption->ntype;
-  pDnode->rebootTime = taosGetTimestampMs();
-
-  if (pDnode->dataDir == NULL || pDnode->localEp == NULL || pDnode->localFqdn == NULL || pDnode->firstEp == NULL ||
-      pDnode->secondEp == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-  return 0;
-}
-
-static void dndClearVars(SDnode *pDnode) {
-  for (ENodeType n = 0; n < NODE_MAX; ++n) {
-    SMgmtWrapper *pMgmt = &pDnode->wrappers[n];
-    taosMemoryFreeClear(pMgmt->path);
-  }
-  dndCloseRuntimeFile(pDnode);
-  taosMemoryFreeClear(pDnode->localEp);
-  taosMemoryFreeClear(pDnode->localFqdn);
-  taosMemoryFreeClear(pDnode->firstEp);
-  taosMemoryFreeClear(pDnode->secondEp);
-  taosMemoryFreeClear(pDnode->dataDir);
-  taosMemoryFree(pDnode);
-  dDebug("dnode object memory is cleared, data:%p", pDnode);
-}
-
-SDnode *dndCreate(const SDnodeOpt *pOption) {
-  dInfo("start to create dnode object");
-  int32_t code = -1;
-  char    path[PATH_MAX] = {0};
-  SDnode *pDnode = NULL;
-
-  pDnode = taosMemoryCalloc(1, sizeof(SDnode));
-  if (pDnode == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    goto _OVER;
-  }
-
-  if (dndInitVars(pDnode, pOption) != 0) {
-    dError("failed to init variables since %s", terrstr());
-    goto _OVER;
-  }
-
-  dndSetStatus(pDnode, DND_STAT_INIT);
-  dmGetMgmtFp(&pDnode->wrappers[DNODE]);
-  mmGetMgmtFp(&pDnode->wrappers[MNODE]);
-  vmGetMgmtFp(&pDnode->wrappers[VNODES]);
-  qmGetMgmtFp(&pDnode->wrappers[QNODE]);
-  smGetMgmtFp(&pDnode->wrappers[SNODE]);
-  bmGetMgmtFp(&pDnode->wrappers[BNODE]);
-
-  if (dndOpenRuntimeFile(pDnode) != 0) {
-    dError("failed to open runtime file since %s", terrstr());
-    goto _OVER;
-  }
-
-  if (dndInitServer(pDnode) != 0) {
-    dError("failed to init trans server since %s", terrstr());
-    goto _OVER;
-  }
-
-  if (dndInitClient(pDnode) != 0) {
-    dError("failed to init trans client since %s", terrstr());
-    goto _OVER;
-  }
-
-  if (dndInitMsgHandle(pDnode) != 0) {
-    goto _OVER;
-  }
-
-  for (ENodeType n = 0; n < NODE_MAX; ++n) {
-    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
-    snprintf(path, sizeof(path), "%s%s%s", pDnode->dataDir, TD_DIRSEP, pWrapper->name);
-    pWrapper->path = strdup(path);
-    pWrapper->pDnode = pDnode;
-    if (pWrapper->path == NULL) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      goto _OVER;
-    }
-
-    pWrapper->procType = PROC_SINGLE;
-    taosInitRWLatch(&pWrapper->latch);
-  }
-
-  code = 0;
-
-_OVER:
-  if (code != 0 && pDnode) {
-    dndClearVars(pDnode);
-    pDnode = NULL;
-    dError("failed to create dnode object since %s", terrstr());
-  } else {
-    dInfo("dnode object is created, data:%p", pDnode);
-  }
-
-  return pDnode;
-}
-
-void dndClose(SDnode *pDnode) {
-  if (pDnode == NULL) return;
-
-  if (dndGetStatus(pDnode) == DND_STAT_STOPPED) {
-    dError("dnode is shutting down, data:%p", pDnode);
-    return;
-  }
-
-  dInfo("start to close dnode, data:%p", pDnode);
-  dndSetStatus(pDnode, DND_STAT_STOPPED);
-
-  dndCleanupServer(pDnode);
-  dndCleanupClient(pDnode);
-
-  for (ENodeType n = 0; n < NODE_MAX; ++n) {
-    SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
-    dndCloseNode(pWrapper);
-  }
-
-  dndClearVars(pDnode);
-  dInfo("dnode object is closed, data:%p", pDnode);
-}
-
-void dndHandleEvent(SDnode *pDnode, EDndEvent event) {
-  dInfo("dnode object receive event %d, data:%p", event, pDnode);
-  pDnode->event = event;
-}
-
-SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType ntype) {
-  SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
-  SMgmtWrapper *pRetWrapper = pWrapper;
-
-  taosRLockLatch(&pWrapper->latch);
-  if (pWrapper->deployed) {
-    int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1);
-    dTrace("node:%s, is acquired, refCount:%d", pWrapper->name, refCount);
-  } else {
-    terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
-    pRetWrapper = NULL;
-  }
-  taosRUnLockLatch(&pWrapper->latch);
-
-  return pRetWrapper;
-}
-
-int32_t dndMarkWrapper(SMgmtWrapper *pWrapper) {
-  int32_t code = 0;
-
-  taosRLockLatch(&pWrapper->latch);
-  if (pWrapper->deployed || (pWrapper->procType == PROC_PARENT && pWrapper->required)) {
-    int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1);
-    dTrace("node:%s, is marked, refCount:%d", pWrapper->name, refCount);
-  } else {
-    terrno = TSDB_CODE_NODE_NOT_DEPLOYED;
-    code = -1;
-  }
-  taosRUnLockLatch(&pWrapper->latch);
-
-  return code;
-}
-
-void dndReleaseWrapper(SMgmtWrapper *pWrapper) {
-  if (pWrapper == NULL) return;
-
-  taosRLockLatch(&pWrapper->latch);
-  int32_t refCount = atomic_sub_fetch_32(&pWrapper->refCount, 1);
-  taosRUnLockLatch(&pWrapper->latch);
-  dTrace("node:%s, is released, refCount:%d", pWrapper->name, refCount);
-}
\ No newline at end of file

source/dnode/mgmt/main/src/dndStr.c

@@ -62,3 +62,16 @@ const char *dndNodeProcStr(ENodeType ntype) {
       return "taosd";
   }
 }
+
+const char *dndEventStr(EDndEvent ev) {
+  switch (ev) {
+    case DND_EVENT_START:
+      return "start";
+    case DND_EVENT_STOP:
+      return "stop";
+    case DND_EVENT_CHILD:
+      return "child";
+    default:
+      return "UNKNOWN";
+  }
+}
\ No newline at end of file

source/dnode/mgmt/vm/src/vmWorker.c

@@ -187,7 +187,7 @@ static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueTyp
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
   if (pVnode == NULL) {
     dError("vgId:%d, failed to write msg:%p to vnode-queue since %s", pHead->vgId, pMsg, terrstr());
-    return -1;
+    return terrno;
   }
 
   int32_t code = 0;

source/dnode/mnode/impl/inc/mndDef.h

@@ -735,6 +735,9 @@ typedef struct {
   int8_t         createdBy;      // STREAM_CREATED_BY__USER or SMA
   int32_t        fixedSinkVgId;  // 0 for shuffle
   int64_t        smaId;          // 0 for unused
+  int8_t         trigger;
+  int32_t        triggerParam;
+  int64_t        waterMark;
   char*          sql;
   char*          logicalPlan;
   char*          physicalPlan;

source/dnode/mnode/impl/src/mndDb.c

@@ -1309,7 +1309,7 @@ static int32_t mndGetDbMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMet
   cols++;
 
   pShow->bytes[cols] = 2;
-  pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
   strcpy(pSchema[cols].name, "days");
   pSchema[cols].bytes = pShow->bytes[cols];
   cols++;
@@ -1444,7 +1444,7 @@ static void dumpDbInfoToPayload(char *data, SDbObj *pDb, SShowObj *pShow, int32_
   cols++;
 
   pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);
-  *(int16_t *)pWrite = pDb->cfg.daysPerFile;
+  *(int32_t *)pWrite = pDb->cfg.daysPerFile;
   cols++;
 
   pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity);

source/dnode/mnode/impl/src/mndInfoSchema.c

@@ -53,7 +53,7 @@ static const SInfosTableSchema userDBSchema[] = {
     {.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
     {.name = "replica", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
     {.name = "quorum", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
-    {.name = "days", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
+    {.name = "days", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "keep", .bytes = 24 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "cache", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
@@ -89,7 +89,6 @@ static const SInfosTableSchema userStbsSchema[] = {
     {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
     {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "last_update", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
     {.name = "table_comment", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_INT},
 };

source/dnode/mnode/impl/src/mndShow.c

@@ -310,6 +310,8 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) {
       mError("failed to process show-retrieve req:%p since %s", pShow, terrstr());
       return -1;
     }
+
+    pShow->numOfReads = 0;
   }
 
   ShowRetrieveFp retrieveFp = pMgmt->retrieveFps[pShow->type];
@@ -374,7 +376,7 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) {
   pReq->pRsp = pRsp;
   pReq->rspLen = size;
 
-  if (rowsRead == 0 || rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) {
+  if (rowsRead == 0 || rowsToRead == 0 || (rowsRead < rowsToRead)) {
     pRsp->completed = 1;
     mDebug("show:0x%" PRIx64 ", retrieve completed", pShow->id);
     mndReleaseShowObj((SShowObj*) pShow, true);

source/dnode/mnode/impl/src/mndStb.c

@@ -1608,7 +1608,6 @@ static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, char *data, int32
   SStbObj *pStb = NULL;
   int32_t  cols = 0;
   char    *pWrite;
-  char     prefix[TSDB_DB_FNAME_LEN] = {0};
 
   SDbObj* pDb = NULL;
   if (strlen(pShow->db) > 0) {
@@ -1653,10 +1652,6 @@ static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, char *data, int32
     *(int32_t *)pWrite = pStb->numOfTags;
     cols++;
 
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int32_t *)pWrite = 0; // number of tables
-    cols++;
-
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
     *(int64_t *)pWrite = pStb->updateTime; // number of tables
     cols++;

source/dnode/mnode/impl/test/sma/sma.cpp

@@ -42,10 +42,10 @@ void* MndTestSma::BuildCreateDbReq(const char* dbname, int32_t* pContLen) {
   createReq.numOfVgroups = 2;
   createReq.cacheBlockSize = 16;
   createReq.totalBlocks = 10;
-  createReq.daysPerFile = 10;
-  createReq.daysToKeep0 = 3650;
-  createReq.daysToKeep1 = 3650;
-  createReq.daysToKeep2 = 3650;
+  createReq.daysPerFile = 10 * 1440;
+  createReq.daysToKeep0 = 3650 * 1440;
+  createReq.daysToKeep1 = 3650 * 1440;
+  createReq.daysToKeep2 = 3650 * 1440;
   createReq.minRows = 100;
   createReq.maxRows = 4096;
   createReq.commitTime = 3600;

source/dnode/mnode/impl/test/topic/topic.cpp

@@ -35,10 +35,10 @@ void* MndTestTopic::BuildCreateDbReq(const char* dbname, int32_t* pContLen) {
   createReq.numOfVgroups = 2;
   createReq.cacheBlockSize = 16;
   createReq.totalBlocks = 10;
-  createReq.daysPerFile = 10;
-  createReq.daysToKeep0 = 3650;
-  createReq.daysToKeep1 = 3650;
-  createReq.daysToKeep2 = 3650;
+  createReq.daysPerFile = 10 * 1440;
+  createReq.daysToKeep0 = 3650 * 1440;
+  createReq.daysToKeep1 = 3650 * 1440;
+  createReq.daysToKeep2 = 3650 * 1440;
   createReq.minRows = 100;
   createReq.maxRows = 4096;
   createReq.commitTime = 3600;

source/dnode/mnode/impl/test/user/user.cpp

@@ -324,10 +324,10 @@ TEST_F(MndTestUser, 03_Alter_User) {
     createReq.numOfVgroups = 2;
     createReq.cacheBlockSize = 16;
     createReq.totalBlocks = 10;
-    createReq.daysPerFile = 10;
-    createReq.daysToKeep0 = 3650;
-    createReq.daysToKeep1 = 3650;
-    createReq.daysToKeep2 = 3650;
+    createReq.daysPerFile = 10 * 1440;
+    createReq.daysToKeep0 = 3650 * 1440;
+    createReq.daysToKeep1 = 3650 * 1440;
+    createReq.daysToKeep2 = 3650 * 1440;
     createReq.minRows = 100;
     createReq.maxRows = 4096;
     createReq.commitTime = 3600;

source/dnode/vnode/src/inc/tsdbSma.h

@@ -51,7 +51,7 @@ static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void
   return len;
 }
 
-static FORCE_INLINE int tsdbRLockSma(SSmaEnv *pEnv) {
+static FORCE_INLINE int32_t tsdbRLockSma(SSmaEnv *pEnv) {
   int code = taosThreadRwlockRdlock(&(pEnv->lock));
   if (code != 0) {
     terrno = TAOS_SYSTEM_ERROR(code);
@@ -60,7 +60,7 @@ static FORCE_INLINE int tsdbRLockSma(SSmaEnv *pEnv) {
   return 0;
 }
 
-static FORCE_INLINE int tsdbWLockSma(SSmaEnv *pEnv) {
+static FORCE_INLINE int32_t tsdbWLockSma(SSmaEnv *pEnv) {
   int code = taosThreadRwlockWrlock(&(pEnv->lock));
   if (code != 0) {
     terrno = TAOS_SYSTEM_ERROR(code);
@@ -69,7 +69,7 @@ static FORCE_INLINE int tsdbWLockSma(SSmaEnv *pEnv) {
   return 0;
 }
 
-static FORCE_INLINE int tsdbUnLockSma(SSmaEnv *pEnv) {
+static FORCE_INLINE int32_t tsdbUnLockSma(SSmaEnv *pEnv) {
   int code = taosThreadRwlockUnlock(&(pEnv->lock));
   if (code != 0) {
     terrno = TAOS_SYSTEM_ERROR(code);

source/dnode/vnode/src/inc/vnd.h

@@ -194,7 +194,7 @@ void tqClose(STQ*);
 int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version);
 int tqCommit(STQ*);
 
-int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg);
+int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
 int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
 int32_t tqProcessRebReq(STQ* pTq, char* msg);
 int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);

source/dnode/vnode/src/meta/metaTbUid.c

@@ -27,5 +27,5 @@ void metaCloseUidGnrt(SMeta *pMeta) { /* TODO */
 
 tb_uid_t metaGenerateUid(SMeta *pMeta) {
   // Generate a new table UID
-  return ++(pMeta->uidGnrt.nextUid);
+  return tGenIdPI32();
 }

source/dnode/vnode/src/tq/tq.c

@@ -250,7 +250,7 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
   return 0;
 }
 
-int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
+int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   SMqPollReq* pReq = pMsg->pCont;
   int64_t     consumerId = pReq->consumerId;
   int64_t     fetchOffset;
@@ -264,6 +264,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
     fetchOffset = pReq->currentOffset + 1;
   }
 
+  /*printf("tmq poll vg %d req %ld %ld\n", pTq->pVnode->vgId, pReq->currentOffset, fetchOffset);*/
+
   SMqPollRsp rsp = {
       /*.consumerId = consumerId,*/
       .numOfTopics = 0,
@@ -288,37 +290,47 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
   rsp.reqOffset = pReq->currentOffset;
   rsp.skipLogNum = 0;
 
-  SWalHead* pHead;
   while (1) {
     /*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/
-    if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {
+    SWalReadHead* pHead;
+    if (walReadWithHandle_s(pTopic->pReadhandle, fetchOffset, &pHead) < 0) {
       // TODO: no more log, set timer to wait blocking time
       // if data inserted during waiting, launch query and
       // response to user
       break;
     }
-    int8_t pos = fetchOffset % TQ_BUFFER_SIZE;
-    pHead = pTopic->pReadhandle->pHead;
-    if (pHead->head.msgType == TDMT_VND_SUBMIT) {
-      SSubmitReq* pCont = (SSubmitReq*)&pHead->head.body;
-      qTaskInfo_t task = pTopic->buffer.output[pos].task;
+    /*printf("vg %d offset %ld msgType %d from epoch %d\n", pTq->pVnode->vgId, fetchOffset, pHead->msgType,
+     * pReq->epoch);*/
+    /*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/
+    /*pHead = pTopic->pReadhandle->pHead;*/
+    if (pHead->msgType == TDMT_VND_SUBMIT) {
+      SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
+      /*printf("from topic %s from consumer\n", pTopic->topicName, consumerId);*/
+      qTaskInfo_t task = pTopic->buffer.output[workerId].task;
+      ASSERT(task);
       qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK);
       SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
       while (1) {
-        SSDataBlock* pDataBlock;
+        SSDataBlock* pDataBlock = NULL;
         uint64_t     ts;
         if (qExecTask(task, &pDataBlock, &ts) < 0) {
           ASSERT(false);
         }
         if (pDataBlock == NULL) {
-          fetchOffset++;
-          pos = fetchOffset % TQ_BUFFER_SIZE;
-          rsp.skipLogNum++;
+          /*pos = fetchOffset % TQ_BUFFER_SIZE;*/
           break;
         }
 
         taosArrayPush(pRes, pDataBlock);
-        rsp.schema = pTopic->buffer.output[pos].pReadHandle->pSchemaWrapper;
+      }
+
+      if (taosArrayGetSize(pRes) == 0) {
+        fetchOffset++;
+        rsp.skipLogNum++;
+        taosArrayDestroy(pRes);
+        continue;
+      }
+      rsp.schema = pTopic->buffer.output[workerId].pReadHandle->pSchemaWrapper;
       rsp.rspOffset = fetchOffset;
 
       rsp.numOfTopics = 1;
@@ -328,6 +340,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
       void*   buf = rpcMallocCont(tlen);
       if (buf == NULL) {
         pMsg->code = -1;
+        taosMemoryFree(pHead);
         return -1;
       }
       ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
@@ -340,10 +353,14 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
       pMsg->pCont = buf;
       pMsg->contLen = tlen;
       pMsg->code = 0;
+      /*printf("vg %d offset %ld msgType %d from epoch %d actual rsp\n", pTq->pVnode->vgId, fetchOffset,
+       * pHead->msgType,*/
+      /*pReq->epoch);*/
       tmsgSendRsp(pMsg);
+      taosMemoryFree(pHead);
       return 0;
-      }
     } else {
+      taosMemoryFree(pHead);
       fetchOffset++;
       rsp.skipLogNum++;
     }
@@ -368,6 +385,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
   pMsg->contLen = tlen;
   pMsg->code = 0;
   tmsgSendRsp(pMsg);
+  /*printf("vg %d offset %ld from epoch %d not rsp\n", pTq->pVnode->vgId, fetchOffset, pReq->epoch);*/
   /*}*/
 
   return 0;
@@ -432,7 +450,9 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
     };
     pTopic->buffer.output[i].pReadHandle = pReadHandle;
     pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle);
+    ASSERT(pTopic->buffer.output[i].task);
   }
+  printf("set topic %s to consumer %ld\n", pTopic->topicName, req.consumerId);
   taosArrayPush(pConsumer->topics, pTopic);
   tqHandleMovePut(pTq->tqMeta, req.consumerId, pConsumer);
   tqHandleCommit(pTq->tqMeta, req.consumerId);

source/dnode/vnode/src/tq/tqRead.c

@@ -167,7 +167,8 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
       if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) {
         break;
       }
-      if (colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) {
+      if (colDataAppend(pColData, curRow, sVal.val, false) < 0) {
+        /*if (colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) {*/
         taosArrayDestroyEx(pArray, (void (*)(void*))tDeleteSSDataBlock);
         return NULL;
       }

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -1394,7 +1394,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
 
   tsdbDebug("vgId:%d uid:%" PRId64 " a block of data is written to file %s, offset %" PRId64
             " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64,
-            REPO_ID(pRepo), TABLE_TID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len,
+            REPO_ID(pRepo), TABLE_UID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len,
             pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast);
 
   return 0;

source/dnode/vnode/src/tsdb/tsdbSma.c

@@ -651,9 +651,8 @@ static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) {
  */
 static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen) {
   SDBFile *pDBFile = &pSmaH->dFile;
-  tsdbDebug("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", dataLen %d",
-            REPO_ID(pSmaH->pTsdb), pDBFile->path, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8),
-            *(int64_t *)POINTER_SHIFT(smaKey, 10), dataLen);
+  printf("\nvgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 "\n",
+         REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
 
   // TODO: insert sma data blocks into B+Tree(TDB)
   if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen) != 0) {
@@ -874,7 +873,6 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
 
   // key: skey + groupId
   char    smaKey[SMA_KEY_LEN] = {0};
-  void   *pSmaKey = &smaKey;
   char    dataBuf[512] = {0};
   void   *pDataBuf = &dataBuf;
   int32_t sz = taosArrayGetSize(pDataBlocks);
@@ -887,6 +885,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
     for (int32_t j = 0; j < rows; ++j) {
       printf("|");
       TSKEY   skey = TSKEY_INITIAL_VAL;  // the start key of TS window by interval
+      void   *pSmaKey = &smaKey;
       int32_t tlen = 0;
       for (int32_t k = 0; k < colNum; ++k) {
         SColumnInfoData *pColInfoData = *(SColumnInfoData **)taosArrayGet(pDataBlock->pDataBlock, k);
@@ -894,7 +893,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
         switch (pColInfoData->info.type) {
           case TSDB_DATA_TYPE_TIMESTAMP:
             skey = *(TSKEY *)var;
-            printf(" skey = %" PRIi64 " groupId = %" PRId64 "|", skey, groupId);
+            printf("==> skey = %" PRIi64 " groupId = %" PRId64 "|", skey, groupId);
             tsdbEncodeTSmaKey(groupId, skey, &pSmaKey);
             break;
           case TSDB_DATA_TYPE_BOOL:
@@ -976,7 +975,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char
           }
         }
 
-        if (tsdbInsertTSmaBlocks(&tSmaH, pSmaKey, SMA_KEY_LEN, pDataBuf, tlen) != 0) {
+        if (tsdbInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, pDataBuf, tlen) != 0) {
           tsdbWarn("vgId:%d insert tSma data blocks failed for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64
                    " since %s",
                    REPO_ID(pTsdb), indexUid, skey, groupId, tstrerror(terrno));
@@ -1313,17 +1312,14 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid,
   int64_t queryGroupId = 1;
   tsdbEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey);
 
-  tsdbDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", keyLen %d", REPO_ID(pTsdb),
-            tReadH.dFile.path, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8),
-            *(int64_t *)POINTER_SHIFT(smaKey, 10), SMA_KEY_LEN);
+  tsdbDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", REPO_ID(pTsdb),
+            tReadH.dFile.path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN);
 
   void    *result = NULL;
   uint32_t valueSize = 0;
   if ((result = tsdbGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize)) == NULL) {
-    tsdbWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64
-             " since %s",
-             REPO_ID(pTsdb), indexUid, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8),
-             *(int64_t *)POINTER_SHIFT(smaKey, 10), tstrerror(terrno));
+    tsdbWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s",
+             REPO_ID(pTsdb), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno));
     tsdbCloseDBF(&tReadH.dFile);
     return TSDB_CODE_FAILED;
   }

source/dnode/vnode/src/vnd/vnodeQuery.c

@@ -66,12 +66,12 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
     case TDMT_VND_TABLE_META:
       return vnodeGetTableMeta(pVnode, pMsg);
     case TDMT_VND_CONSUME:
-      return tqProcessPollReq(pVnode->pTq, pMsg);
+      return tqProcessPollReq(pVnode->pTq, pMsg, pInfo->workerId);
     case TDMT_VND_TASK_PIPE_EXEC:
     case TDMT_VND_TASK_MERGE_EXEC:
-      return tqProcessTaskExec(pVnode->pTq, msgstr, msgLen, pInfo->workerId);
+      return tqProcessTaskExec(pVnode->pTq, msgstr, msgLen, 0);
     case TDMT_VND_STREAM_TRIGGER:
-      return tqProcessStreamTrigger(pVnode->pTq, pMsg->pCont, pMsg->contLen, pInfo->workerId);
+      return tqProcessStreamTrigger(pVnode->pTq, pMsg->pCont, pMsg->contLen, 0);
     case TDMT_VND_QUERY_HEARTBEAT:
       return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg);
     default:

source/dnode/vnode/src/vnd/vnodeWrite.c

@@ -165,6 +165,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
       // }
       break;
     case TDMT_VND_SUBMIT:
+      /*printf("vnode %d write data %ld\n", pVnode->vgId, ver);*/
       if (pVnode->config.streamMode == 0) {
         if (tsdbInsertData(pVnode->pTsdb, (SSubmitReq *)ptr, NULL) < 0) {
           // TODO: handle error

source/dnode/vnode/test/tsdbSmaTest.cpp

@@ -59,20 +59,21 @@ TEST(testCase, unionEncodeDecodeTest) {
 
   void   *buf = taosMemoryMalloc(1024);
   void   *pBuf = buf;
+  void   *qBuf = buf;
   int32_t tlen = 0;
-  tlen += taosEncodeFixedU8(&buf, sut.info);
-  tlen += taosEncodeFixedI16(&buf, sut.nBSmaCols);
+  tlen += taosEncodeFixedU8(&pBuf, sut.info);
+  tlen += taosEncodeFixedI16(&pBuf, sut.nBSmaCols);
   for (col_id_t i = 0; i < sut.nBSmaCols; ++i) {
-    tlen += taosEncodeFixedI16(&buf, sut.pBSmaCols[i]);
+    tlen += taosEncodeFixedI16(&pBuf, sut.pBSmaCols[i]);
   }
 
   SUnionTest dut = {0};
-  pBuf = taosDecodeFixedU8(pBuf, &dut.info);
-  pBuf = taosDecodeFixedI16(pBuf, &dut.nBSmaCols);
+  qBuf = taosDecodeFixedU8(qBuf, &dut.info);
+  qBuf = taosDecodeFixedI16(qBuf, &dut.nBSmaCols);
   if (dut.nBSmaCols > 0) {
     dut.pBSmaCols = (col_id_t *)taosMemoryMalloc(dut.nBSmaCols * sizeof(col_id_t));
     for (col_id_t i = 0; i < dut.nBSmaCols; ++i) {
-      pBuf = taosDecodeFixedI16(pBuf, dut.pBSmaCols + i);
+      qBuf = taosDecodeFixedI16(qBuf, dut.pBSmaCols + i);
     }
   } else {
     dut.pBSmaCols = NULL;
@@ -81,13 +82,17 @@ TEST(testCase, unionEncodeDecodeTest) {
   printf("sut.rollup=%" PRIu8 ", type=%" PRIu8 ", info=%" PRIu8 "\n", sut.rollup, sut.type, sut.info);
   printf("dut.rollup=%" PRIu8 ", type=%" PRIu8 ", info=%" PRIu8 "\n", dut.rollup, dut.type, dut.info);
 
-  ASSERT_EQ(sut.rollup, dut.rollup);
-  ASSERT_EQ(sut.type, dut.type);
-  ASSERT_EQ(sut.nBSmaCols, dut.nBSmaCols);
+  EXPECT_EQ(sut.rollup, dut.rollup);
+  EXPECT_EQ(sut.type, dut.type);
+  EXPECT_EQ(sut.nBSmaCols, dut.nBSmaCols);
   for (col_id_t i = 0; i < sut.nBSmaCols; ++i) {
-    ASSERT_EQ(*(col_id_t *)(sut.pBSmaCols + i), sut.pBSmaCols[i]);
-    ASSERT_EQ(*(col_id_t *)(sut.pBSmaCols + i), dut.pBSmaCols[i]);
+    EXPECT_EQ(*(col_id_t *)(sut.pBSmaCols + i), sut.pBSmaCols[i]);
+    EXPECT_EQ(*(col_id_t *)(sut.pBSmaCols + i), dut.pBSmaCols[i]);
   }
+
+  taosMemoryFreeClear(buf);
+  taosMemoryFreeClear(dut.pBSmaCols);
+  taosMemoryFreeClear(sut.pBSmaCols);
 }
 #if 1
 TEST(testCase, tSma_Meta_Encode_Decode_Test) {
@@ -107,37 +112,37 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) {
   uint32_t     bufLen = tEncodeTSmaWrapper(NULL, &tSmaWrapper);
 
   void *buf = taosMemoryCalloc(1, bufLen);
-  ASSERT_NE(buf, nullptr);
+  EXPECT_NE(buf, nullptr);
 
   STSmaWrapper *pSW = (STSmaWrapper *)buf;
   uint32_t      len = tEncodeTSmaWrapper(&buf, &tSmaWrapper);
 
-  ASSERT_EQ(len, bufLen);
+  EXPECT_EQ(len, bufLen);
 
   // decode
   STSmaWrapper dstTSmaWrapper = {0};
   void        *result = tDecodeTSmaWrapper(pSW, &dstTSmaWrapper);
-  ASSERT_NE(result, nullptr);
+  EXPECT_NE(result, nullptr);
 
-  ASSERT_EQ(tSmaWrapper.number, dstTSmaWrapper.number);
+  EXPECT_EQ(tSmaWrapper.number, dstTSmaWrapper.number);
 
   for (int i = 0; i < tSmaWrapper.number; ++i) {
     STSma *pSma = tSmaWrapper.tSma + i;
     STSma *qSma = dstTSmaWrapper.tSma + i;
 
-    ASSERT_EQ(pSma->version, qSma->version);
-    ASSERT_EQ(pSma->intervalUnit, qSma->intervalUnit);
-    ASSERT_EQ(pSma->slidingUnit, qSma->slidingUnit);
-    ASSERT_STRCASEEQ(pSma->indexName, qSma->indexName);
-    ASSERT_EQ(pSma->timezoneInt, qSma->timezoneInt);
-    ASSERT_EQ(pSma->indexUid, qSma->indexUid);
-    ASSERT_EQ(pSma->tableUid, qSma->tableUid);
-    ASSERT_EQ(pSma->interval, qSma->interval);
-    ASSERT_EQ(pSma->sliding, qSma->sliding);
-    ASSERT_EQ(pSma->exprLen, qSma->exprLen);
-    ASSERT_STRCASEEQ(pSma->expr, qSma->expr);
-    ASSERT_EQ(pSma->tagsFilterLen, qSma->tagsFilterLen);
-    ASSERT_STRCASEEQ(pSma->tagsFilter, qSma->tagsFilter);
+    EXPECT_EQ(pSma->version, qSma->version);
+    EXPECT_EQ(pSma->intervalUnit, qSma->intervalUnit);
+    EXPECT_EQ(pSma->slidingUnit, qSma->slidingUnit);
+    EXPECT_STRCASEEQ(pSma->indexName, qSma->indexName);
+    EXPECT_EQ(pSma->timezoneInt, qSma->timezoneInt);
+    EXPECT_EQ(pSma->indexUid, qSma->indexUid);
+    EXPECT_EQ(pSma->tableUid, qSma->tableUid);
+    EXPECT_EQ(pSma->interval, qSma->interval);
+    EXPECT_EQ(pSma->sliding, qSma->sliding);
+    EXPECT_EQ(pSma->exprLen, qSma->exprLen);
+    EXPECT_STRCASEEQ(pSma->expr, qSma->expr);
+    EXPECT_EQ(pSma->tagsFilterLen, qSma->tagsFilterLen);
+    EXPECT_STRCASEEQ(pSma->tagsFilter, qSma->tagsFilter);
   }
 
   // resource release
@@ -173,12 +178,12 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
 
   tSma.exprLen = strlen(expr);
   tSma.expr = (char *)taosMemoryCalloc(1, tSma.exprLen + 1);
-  ASSERT_NE(tSma.expr, nullptr);
+  EXPECT_NE(tSma.expr, nullptr);
   tstrncpy(tSma.expr, expr, tSma.exprLen + 1);
 
   tSma.tagsFilterLen = strlen(tagsFilter);
   tSma.tagsFilter = (char *)taosMemoryCalloc(tSma.tagsFilterLen + 1, 1);
-  ASSERT_NE(tSma.tagsFilter, nullptr);
+  EXPECT_NE(tSma.tagsFilter, nullptr);
   tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1);
 
   SMeta          *pMeta = NULL;
@@ -190,7 +195,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
   pMeta = metaOpen(smaTestDir, pMetaCfg, NULL);
   assert(pMeta != NULL);
   // save index 1
-  ASSERT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0);
+  EXPECT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0);
 
   pSmaCfg->indexUid = indexUid2;
   tstrncpy(pSmaCfg->indexName, smaIndexName2, TSDB_INDEX_NAME_LEN);
@@ -201,7 +206,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
   pSmaCfg->sliding = 5;
 
   // save index 2
-  ASSERT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0);
+  EXPECT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0);
 
   // get value by indexName
   STSma *qSmaCfg = NULL;
@@ -211,8 +216,8 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
   printf("timezone1 = %" PRIi8 "\n", qSmaCfg->timezoneInt);
   printf("expr1 = %s\n", qSmaCfg->expr != NULL ? qSmaCfg->expr : "");
   printf("tagsFilter1 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : "");
-  ASSERT_STRCASEEQ(qSmaCfg->indexName, smaIndexName1);
-  ASSERT_EQ(qSmaCfg->tableUid, tSma.tableUid);
+  EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName1);
+  EXPECT_EQ(qSmaCfg->tableUid, tSma.tableUid);
   tdDestroyTSma(qSmaCfg);
   taosMemoryFreeClear(qSmaCfg);
 
@@ -222,8 +227,8 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
   printf("timezone2 = %" PRIi8 "\n", qSmaCfg->timezoneInt);
   printf("expr2 = %s\n", qSmaCfg->expr != NULL ? qSmaCfg->expr : "");
   printf("tagsFilter2 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : "");
-  ASSERT_STRCASEEQ(qSmaCfg->indexName, smaIndexName2);
-  ASSERT_EQ(qSmaCfg->interval, tSma.interval);
+  EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName2);
+  EXPECT_EQ(qSmaCfg->interval, tSma.interval);
   tdDestroyTSma(qSmaCfg);
   taosMemoryFreeClear(qSmaCfg);
 
@@ -239,25 +244,25 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
     printf("indexName = %s\n", indexName);
     ++indexCnt;
   }
-  ASSERT_EQ(indexCnt, nCntTSma);
+  EXPECT_EQ(indexCnt, nCntTSma);
   metaCloseSmaCurosr(pSmaCur);
 
   // get wrapper by table uid
   STSmaWrapper *pSW = metaGetSmaInfoByTable(pMeta, tbUid);
   assert(pSW != NULL);
-  ASSERT_EQ(pSW->number, nCntTSma);
-  ASSERT_STRCASEEQ(pSW->tSma->indexName, smaIndexName1);
-  ASSERT_EQ(pSW->tSma->timezoneInt, timezone);
-  ASSERT_STRCASEEQ(pSW->tSma->expr, expr);
-  ASSERT_STRCASEEQ(pSW->tSma->tagsFilter, tagsFilter);
-  ASSERT_EQ(pSW->tSma->indexUid, indexUid1);
-  ASSERT_EQ(pSW->tSma->tableUid, tbUid);
-  ASSERT_STRCASEEQ((pSW->tSma + 1)->indexName, smaIndexName2);
-  ASSERT_EQ((pSW->tSma + 1)->timezoneInt, timezone);
-  ASSERT_STRCASEEQ((pSW->tSma + 1)->expr, expr);
-  ASSERT_STRCASEEQ((pSW->tSma + 1)->tagsFilter, tagsFilter);
-  ASSERT_EQ((pSW->tSma + 1)->indexUid, indexUid2);
-  ASSERT_EQ((pSW->tSma + 1)->tableUid, tbUid);
+  EXPECT_EQ(pSW->number, nCntTSma);
+  EXPECT_STRCASEEQ(pSW->tSma->indexName, smaIndexName1);
+  EXPECT_EQ(pSW->tSma->timezoneInt, timezone);
+  EXPECT_STRCASEEQ(pSW->tSma->expr, expr);
+  EXPECT_STRCASEEQ(pSW->tSma->tagsFilter, tagsFilter);
+  EXPECT_EQ(pSW->tSma->indexUid, indexUid1);
+  EXPECT_EQ(pSW->tSma->tableUid, tbUid);
+  EXPECT_STRCASEEQ((pSW->tSma + 1)->indexName, smaIndexName2);
+  EXPECT_EQ((pSW->tSma + 1)->timezoneInt, timezone);
+  EXPECT_STRCASEEQ((pSW->tSma + 1)->expr, expr);
+  EXPECT_STRCASEEQ((pSW->tSma + 1)->tagsFilter, tagsFilter);
+  EXPECT_EQ((pSW->tSma + 1)->indexUid, indexUid2);
+  EXPECT_EQ((pSW->tSma + 1)->tableUid, tbUid);
 
   tdDestroyTSmaWrapper(pSW);
   taosMemoryFreeClear(pSW);
@@ -269,7 +274,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) {
     printf("metaGetSmaTbUids: uid[%" PRIu32 "] = %" PRIi64 "\n", i, *(tb_uid_t *)taosArrayGet(pUids, i));
     // printf("metaGetSmaTbUids: index[%" PRIu32 "] = %s", i, (char *)taosArrayGet(pUids, i));
   }
-  ASSERT_EQ(taosArrayGetSize(pUids), 1);
+  EXPECT_EQ(taosArrayGetSize(pUids), 1);
   taosArrayDestroy(pUids);
 
   // resource release
@@ -311,12 +316,12 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
 
   tSma.exprLen = strlen(expr);
   tSma.expr = (char *)taosMemoryCalloc(1, tSma.exprLen + 1);
-  ASSERT_NE(tSma.expr, nullptr);
+  EXPECT_NE(tSma.expr, nullptr);
   tstrncpy(tSma.expr, expr, tSma.exprLen + 1);
 
   tSma.tagsFilterLen = strlen(tagsFilter);
   tSma.tagsFilter = (char *)taosMemoryCalloc(1, tSma.tagsFilterLen + 1);
-  ASSERT_NE(tSma.tagsFilter, nullptr);
+  EXPECT_NE(tSma.tagsFilter, nullptr);
   tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1);
 
   SMeta          *pMeta = NULL;
@@ -328,7 +333,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
   pMeta = metaOpen(smaTestDir, pMetaCfg, NULL);
   assert(pMeta != NULL);
   // save index 1
-  ASSERT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0);
+  EXPECT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0);
 
   // step 2: insert data
   STsdb    *pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(STsdb));
@@ -365,7 +370,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
   strncpy(pDisks.dir, "/var/lib/taos", TSDB_FILENAME_LEN);
   int32_t numOfDisks = 1;
   pTsdb->pTfs = tfsOpen(&pDisks, numOfDisks);
-  ASSERT_NE(pTsdb->pTfs, nullptr);
+  EXPECT_NE(pTsdb->pTfs, nullptr);
 
   // generate SSubmitReq msg and update expired window
   int16_t  schemaVer = 0;
@@ -375,7 +380,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
   uint32_t msgLen = sizeof(SSubmitReq) + mockBlkNum * sizeof(SSubmitBlk) + mockBlkNum * mockRowNum * mockRowLen;
 
   SSubmitReq *pMsg = (SSubmitReq *)taosMemoryCalloc(1, msgLen);
-  ASSERT_NE(pMsg, nullptr);
+  EXPECT_NE(pMsg, nullptr);
   pMsg->version = htobe64(schemaVer);
   pMsg->numOfBlocks = htonl(mockBlkNum);
   pMsg->length = htonl(msgLen);
@@ -401,9 +406,9 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
     }
   }
 
-  ASSERT_EQ(tdScanAndConvertSubmitMsg(pMsg), TSDB_CODE_SUCCESS);
+  EXPECT_EQ(tdScanAndConvertSubmitMsg(pMsg), TSDB_CODE_SUCCESS);
 
-  ASSERT_EQ(tsdbUpdateSmaWindow(pTsdb, (const char *)pMsg), 0);
+  EXPECT_EQ(tsdbUpdateSmaWindow(pTsdb, (const char *)pMsg), 0);
 
   // init
   const int32_t  tSmaGroupSize = 4;
@@ -413,7 +418,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
   const int32_t  tSmaNumOfRows = 2;
 
   SArray *pDataBlocks = taosArrayInit(tSmaGroupSize, sizeof(SSDataBlock *));
-  ASSERT_NE(pDataBlocks, nullptr);
+  EXPECT_NE(pDataBlocks, nullptr);
   int32_t tSmaTypeArray[tSmaNumOfCols] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL,     TSDB_DATA_TYPE_INT,
                                           TSDB_DATA_TYPE_UBIGINT,   TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_FLOAT,
                                           TSDB_DATA_TYPE_DOUBLE,    TSDB_DATA_TYPE_VARCHAR,  TSDB_DATA_TYPE_NCHAR};
@@ -427,18 +432,18 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
 
   for (int32_t g = 0; g < tSmaGroupSize; ++g) {
     SSDataBlock *pDataBlock = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock));
-    ASSERT_NE(pDataBlock, nullptr);
+    EXPECT_NE(pDataBlock, nullptr);
     pDataBlock->pBlockAgg = NULL;
     pDataBlock->info.numOfCols = tSmaNumOfCols;
     pDataBlock->info.rows = tSmaNumOfRows;
     pDataBlock->info.groupId = tSmaGroupId + g;
 
     pDataBlock->pDataBlock = taosArrayInit(tSmaNumOfCols, sizeof(SColumnInfoData *));
-    ASSERT_NE(pDataBlock->pDataBlock, nullptr);
+    EXPECT_NE(pDataBlock->pDataBlock, nullptr);
     for (int32_t c = 0; c < tSmaNumOfCols; ++c) {
       
       SColumnInfoData *pColInfoData = (SColumnInfoData *)taosMemoryCalloc(1, sizeof(SColumnInfoData));
-      ASSERT_NE(pColInfoData, nullptr);
+      EXPECT_NE(pColInfoData, nullptr);
 
       pColInfoData->info.type = tSmaTypeArray[c];
       if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
@@ -481,7 +486,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
             memcpy(varDataVal(pCellData), tSmaGroupbyTags[g * 2 + 1], varDataLen(pCellData));
             break;
           default:
-            ASSERT_EQ(0, 1);  // add definition
+            EXPECT_EQ(0, 1);  // add definition
             break;
         }
       }
@@ -493,7 +498,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
   }
 
   // execute
-  ASSERT_EQ(tsdbInsertTSmaData(pTsdb, tSma.indexUid, (const char *)pDataBlocks), TSDB_CODE_SUCCESS);
+  EXPECT_EQ(tsdbInsertTSmaData(pTsdb, tSma.indexUid, (const char *)pDataBlocks), TSDB_CODE_SUCCESS);
 
 #if 0
   STSmaDataWrapper *pSmaData = NULL;
@@ -512,7 +517,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
     int32_t tableDataLen = sizeof(STSmaTbData);
     for (col_id_t c = 0; c < numOfCols; ++c) {
       if (bufSize - len - tableDataLen < buffer) {
-        ASSERT_EQ(tsdbMakeRoom(&buf, bufSize + allocStep), 0);
+        EXPECT_EQ(tsdbMakeRoom(&buf, bufSize + allocStep), 0);
         pSmaData = (STSmaDataWrapper *)buf;
         pTbData = (STSmaTbData *)POINTER_SHIFT(pSmaData, len);
         bufSize = taosTSizeof(buf);
@@ -539,14 +544,14 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
   }
   pSmaData->dataLen = (len - sizeof(STSmaDataWrapper));
 
-  ASSERT_GE(bufSize, pSmaData->dataLen);
+  EXPECT_GE(bufSize, pSmaData->dataLen);
   // execute
-  ASSERT_EQ(tsdbInsertTSmaData(pTsdb, (char *)pSmaData), TSDB_CODE_SUCCESS);
+  EXPECT_EQ(tsdbInsertTSmaData(pTsdb, (char *)pSmaData), TSDB_CODE_SUCCESS);
 #endif
 
   // step 3: query
   uint32_t checkDataCnt = 0;
-  ASSERT_EQ(tsdbGetTSmaData(pTsdb, NULL, indexUid1, skey1, 1), TSDB_CODE_SUCCESS);
+  EXPECT_EQ(tsdbGetTSmaData(pTsdb, NULL, indexUid1, skey1, 1), TSDB_CODE_SUCCESS);
   ++checkDataCnt;
 
   printf("%s:%d The sma data check count for insert and query is %" PRIu32 "\n", __FILE__, __LINE__, checkDataCnt);
@@ -555,11 +560,12 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
   taosMemoryFreeClear(pMsg);
 
   for (int32_t i = 0; i < taosArrayGetSize(pDataBlocks); ++i) {
-    SSDataBlock *pDataBlock = (SSDataBlock *)taosArrayGet(pDataBlocks, i);
+    SSDataBlock *pDataBlock = *(SSDataBlock **)taosArrayGet(pDataBlocks, i);
     int32_t      numOfOutput = taosArrayGetSize(pDataBlock->pDataBlock);
     for (int32_t j = 0; j < numOfOutput; ++j) {
-      SColumnInfoData *pColInfoData = (SColumnInfoData *)taosArrayGet(pDataBlock->pDataBlock, j);
+      SColumnInfoData *pColInfoData = *(SColumnInfoData **)taosArrayGet(pDataBlock->pDataBlock, j);
       colDataDestroy(pColInfoData);
+      taosMemoryFreeClear(pColInfoData);
     }
 
     taosArrayDestroy(pDataBlock->pDataBlock);

source/libs/CMakeLists.txt

@@ -17,3 +17,4 @@ add_subdirectory(tfs)
 add_subdirectory(monitor)
 add_subdirectory(nodes)
 add_subdirectory(scalar)
+add_subdirectory(command)

source/libs/catalog/inc/catalogInt.h

@@ -112,7 +112,14 @@ typedef struct SCtgRuntimeStat {
 } SCtgRuntimeStat;
 
 typedef struct SCtgCacheStat {
-
+  uint64_t clusterNum;
+  uint64_t dbNum;
+  uint64_t tblNum;
+  uint64_t stblNum;
+  uint64_t vgHitNum;
+  uint64_t vgMissNum;
+  uint64_t tblHitNum;
+  uint64_t tblMissNum;
 } SCtgCacheStat;
 
 typedef struct SCatalogStat {
@@ -204,8 +211,13 @@ typedef struct SCtgAction {
 #define CTG_QUEUE_ADD() atomic_add_fetch_64(&gCtgMgmt.queue.qRemainNum, 1)
 #define CTG_QUEUE_SUB() atomic_sub_fetch_64(&gCtgMgmt.queue.qRemainNum, 1)
 
-#define CTG_STAT_ADD(n) atomic_add_fetch_64(&(n), 1)
-#define CTG_STAT_SUB(n) atomic_sub_fetch_64(&(n), 1)
+#define CTG_STAT_ADD(_item, _n) atomic_add_fetch_64(&(_item), _n)
+#define CTG_STAT_SUB(_item, _n) atomic_sub_fetch_64(&(_item), _n)
+#define CTG_STAT_GET(_item) atomic_load_64(&(_item))
+
+#define CTG_RUNTIME_STAT_ADD(item, n) (CTG_STAT_ADD(gCtgMgmt.stat.runtime.item, n))
+#define CTG_CACHE_STAT_ADD(item, n) (CTG_STAT_ADD(gCtgMgmt.stat.cache.item, n))
+#define CTG_CACHE_STAT_SUB(item, n) (CTG_STAT_SUB(gCtgMgmt.stat.cache.item, n))
 
 #define CTG_IS_META_NULL(type) ((type) == META_TYPE_NULL_TABLE)
 #define CTG_IS_META_CTABLE(type) ((type) == META_TYPE_CTABLE)
@@ -291,6 +303,9 @@ typedef struct SCtgAction {
 #define CTG_API_ENTER() do { CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); }  } while (0)
 
 
+extern void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p);
+extern void ctgdShowClusterCache(SCatalog* pCtg);
+extern int32_t ctgdShowCacheInfo(void);
 
 #ifdef __cplusplus
 }

source/libs/catalog/src/catalogDbg.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "trpc.h"
+#include "query.h"
+#include "tname.h"
+#include "catalogInt.h"
+
+extern SCatalogMgmt gCtgMgmt;
+SCtgDebug gCTGDebug = {0};
+
+int32_t ctgdEnableDebug(char *option) {
+  if (0 == strcasecmp(option, "lock")) {
+    gCTGDebug.lockEnable = true;
+    qDebug("lock debug enabled");
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (0 == strcasecmp(option, "cache")) {
+    gCTGDebug.cacheEnable = true;
+    qDebug("cache debug enabled");
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (0 == strcasecmp(option, "api")) {
+    gCTGDebug.apiEnable = true;
+    qDebug("api debug enabled");
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (0 == strcasecmp(option, "meta")) {
+    gCTGDebug.metaEnable = true;
+    qDebug("api debug enabled");
+    return TSDB_CODE_SUCCESS;
+  }
+
+  qError("invalid debug option:%s", option);
+  
+  return TSDB_CODE_CTG_INTERNAL_ERROR;
+}
+
+int32_t ctgdGetStatNum(char *option, void *res) {
+  if (0 == strcasecmp(option, "runtime.qDoneNum")) {
+    *(uint64_t *)res = atomic_load_64(&gCtgMgmt.stat.runtime.qDoneNum);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  qError("invalid stat option:%s", option);
+  
+  return TSDB_CODE_CTG_INTERNAL_ERROR;
+}
+
+int32_t ctgdGetTbMetaNum(SCtgDBCache *dbCache) {
+  return dbCache->tbCache.metaCache ? (int32_t)taosHashGetSize(dbCache->tbCache.metaCache) : 0;
+}
+
+int32_t ctgdGetStbNum(SCtgDBCache *dbCache) {
+  return dbCache->tbCache.stbCache ? (int32_t)taosHashGetSize(dbCache->tbCache.stbCache) : 0;
+}
+
+int32_t ctgdGetRentNum(SCtgRentMgmt *rent) {
+  int32_t num = 0;
+  for (uint16_t i = 0; i < rent->slotNum; ++i) {
+    SCtgRentSlot *slot = &rent->slots[i];
+    if (NULL == slot->meta) {
+      continue;
+    }
+
+    num += taosArrayGetSize(slot->meta);
+  }
+
+  return num;
+}
+
+int32_t ctgdGetClusterCacheNum(SCatalog* pCtg, int32_t type) {
+  if (NULL == pCtg || NULL == pCtg->dbCache) {
+    return 0;
+  }
+
+  switch (type) {
+    case CTG_DBG_DB_NUM:
+      return (int32_t)taosHashGetSize(pCtg->dbCache);
+    case CTG_DBG_DB_RENT_NUM:
+      return ctgdGetRentNum(&pCtg->dbRent);
+    case CTG_DBG_STB_RENT_NUM:
+      return ctgdGetRentNum(&pCtg->stbRent);
+    default:
+      break;
+  }
+
+  SCtgDBCache *dbCache = NULL;
+  int32_t num = 0;
+  void *pIter = taosHashIterate(pCtg->dbCache, NULL);
+  while (pIter) {
+    dbCache = (SCtgDBCache *)pIter;
+    switch (type) {
+      case CTG_DBG_META_NUM:
+        num += ctgdGetTbMetaNum(dbCache);
+        break;
+      case CTG_DBG_STB_NUM:
+        num += ctgdGetStbNum(dbCache);
+        break;
+      default:
+        ctgError("invalid type:%d", type);
+        break;
+    }
+    pIter = taosHashIterate(pCtg->dbCache, pIter);
+  }
+
+  return num;
+}
+
+void ctgdShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) {
+  if (!gCTGDebug.metaEnable) {
+    return;
+  }
+
+  STableComInfo *c = &p->tableInfo;
+
+  if (TSDB_CHILD_TABLE == p->tableType) {
+    ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64, tbName, p->tableType, p->vgId, p->uid, p->suid);
+    return;
+  } else {
+    ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d",
+     tbName, p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize);
+  }
+
+  int32_t colNum = c->numOfColumns + c->numOfTags;
+  for (int32_t i = 0; i < colNum; ++i) {
+    SSchema *s = &p->schema[i];
+    ctgDebug("[%d] name:%s, type:%d, colId:%d, bytes:%d", i, s->name, s->type, s->colId, s->bytes);
+  }
+}
+
+void ctgdShowDBCache(SCatalog* pCtg, SHashObj *dbHash) {
+  if (NULL == dbHash || !gCTGDebug.cacheEnable) {
+    return;
+  }
+
+  int32_t i = 0;
+  SCtgDBCache *dbCache = NULL;
+  void *pIter = taosHashIterate(dbHash, NULL);
+  while (pIter) {
+    char *dbFName = NULL;
+    size_t len = 0;
+    
+    dbCache = (SCtgDBCache *)pIter;
+
+    dbFName = taosHashGetKey(pIter, &len);
+
+    int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0;
+    int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0;
+    int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION;
+    int32_t hashMethod = -1;
+    int32_t vgNum = 0;
+
+    if (dbCache->vgInfo) {
+      vgVersion = dbCache->vgInfo->vgVersion;
+      hashMethod = dbCache->vgInfo->hashMethod;
+      if (dbCache->vgInfo->vgHash) {
+        vgNum = taosHashGetSize(dbCache->vgInfo->vgHash);
+      }
+    }
+    
+    ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d",
+      i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, vgNum);
+
+    pIter = taosHashIterate(dbHash, pIter);
+  }
+}
+
+
+
+
+void ctgdShowClusterCache(SCatalog* pCtg) {
+  if (!gCTGDebug.cacheEnable || NULL == pCtg) {
+    return;
+  }
+
+  ctgDebug("## cluster %"PRIx64" %p cache Info BEGIN ##", pCtg->clusterId, pCtg);
+  ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM),
+    ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM));    
+  
+  ctgdShowDBCache(pCtg, pCtg->dbCache);
+
+  ctgDebug("## cluster %"PRIx64" %p cache Info END ##", pCtg->clusterId, pCtg);
+}
+
+int32_t ctgdShowCacheInfo(void) {
+  if (!gCTGDebug.cacheEnable) {
+    return TSDB_CODE_CTG_OUT_OF_SERVICE;
+  }
+
+  CTG_API_ENTER();
+  
+  SCatalog *pCtg = NULL;
+  void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL);
+  while (pIter) {
+    pCtg = *(SCatalog **)pIter;
+
+    if (pCtg) {
+      ctgdShowClusterCache(pCtg);
+    }
+    
+    pIter = taosHashIterate(gCtgMgmt.pCluster, pIter);
+  }
+
+  CTG_API_LEAVE(TSDB_CODE_SUCCESS);
+}
+

source/libs/catalog/test/catalogTests.cpp

@@ -38,11 +38,11 @@
 namespace {
 
 extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta,
-                                            int32_t *exist, int32_t flag, uint64_t *dbId);
-extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type);
+                                            bool *inCache, int32_t flag, uint64_t *dbId);
+extern "C" int32_t ctgdGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type);
 extern "C" int32_t ctgActUpdateTbl(SCtgMetaAction *action);
-extern "C" int32_t ctgDbgEnableDebug(char *option);
-extern "C" int32_t ctgDbgGetStatNum(char *option, void *res);
+extern "C" int32_t ctgdEnableDebug(char *option);
+extern "C" int32_t ctgdGetStatNum(char *option, void *res);
 
 void ctgTestSetRspTableMeta();
 void ctgTestSetRspCTableMeta();
@@ -140,9 +140,9 @@ void ctgTestInitLogFile() {
   qDebugFlag = 159;
   strcpy(tsLogDir, "/var/log/taos");
 
-  ctgDbgEnableDebug("api");
-  ctgDbgEnableDebug("meta");
-  ctgDbgEnableDebug("cache");
+  ctgdEnableDebug("api");
+  ctgdEnableDebug("meta");
+  ctgdEnableDebug("cache");
   
   if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
     printf("failed to open log file in directory:%s\n", tsLogDir);
@@ -786,15 +786,15 @@ void *ctgTestGetCtableMetaThread(void *param) {
   int32_t          code = 0;
   int32_t          n = 0;
   STableMeta      *tbMeta = NULL;
-  int32_t          exist = 0;
+  bool             inCache = false;
 
   SName cn = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
   strcpy(cn.dbname, "db1");
   strcpy(cn.tname, ctgTestCTablename);
 
   while (!ctgTestStop) {
-    code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &exist, 0, NULL);
-    if (code || 0 == exist) {
+    code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &inCache, 0, NULL);
+    if (code || !inCache) {
       assert(0);
     }
 
@@ -879,7 +879,7 @@ TEST(tableMeta, normalTable) {
   ASSERT_EQ(vgInfo.vgId, 8);
   ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
 
-  while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) {
+  while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) {
     taosMsleep(50);
   }
   
@@ -899,7 +899,7 @@ TEST(tableMeta, normalTable) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
   while (true) {
-    uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
     if (0 == n) {
       taosMsleep(50);
     } else {
@@ -994,7 +994,7 @@ TEST(tableMeta, childTableCase) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
   while (true) {
-    uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
     if (0 == n) {
       taosMsleep(50);
     } else {
@@ -1103,7 +1103,7 @@ TEST(tableMeta, superTableCase) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
   while (true) {
-    uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
     if (0 == n) {
       taosMsleep(50);
     } else {
@@ -1130,7 +1130,7 @@ TEST(tableMeta, superTableCase) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
   while (true) {
-    uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
     if (2 != n) {
       taosMsleep(50);
     } else {
@@ -1228,7 +1228,7 @@ TEST(tableMeta, rmStbMeta) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
   while (true) {
-    uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
     if (0 == n) {
       taosMsleep(50);
     } else {
@@ -1241,8 +1241,8 @@ TEST(tableMeta, rmStbMeta) {
   ASSERT_EQ(code, 0);
 
   while (true) {
-    int32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
-    int32_t m = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM);
+    int32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    int32_t m = ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM);
     if (n || m) {
       taosMsleep(50);
     } else {
@@ -1251,11 +1251,11 @@ TEST(tableMeta, rmStbMeta) {
   }
 
 
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 0);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 0);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0);
 
   catalogDestroy();
   memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
@@ -1298,7 +1298,7 @@ TEST(tableMeta, updateStbMeta) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
   while (true) {
-    uint32_t n = ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
     if (0 == n) {
       taosMsleep(50);
     } else {
@@ -1318,7 +1318,7 @@ TEST(tableMeta, updateStbMeta) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n != 3) {
       taosMsleep(50);
     } else {
@@ -1326,11 +1326,11 @@ TEST(tableMeta, updateStbMeta) {
     }
   }
 
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 1);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 1);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1);
-  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 1);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 1);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 1);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1);
+  ASSERT_EQ(ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 1);
 
   code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta);
   ASSERT_EQ(code, 0);
@@ -1388,7 +1388,7 @@ TEST(refreshGetMeta, normal2normal) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n > 0) {
       break;
     }
@@ -1409,7 +1409,7 @@ TEST(refreshGetMeta, normal2normal) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
   taosMemoryFreeClear(tableMeta);
 
-  while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
+  while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
     taosMsleep(50);
   }
 
@@ -1467,7 +1467,7 @@ TEST(refreshGetMeta, normal2notexist) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n > 0) {
       break;
     }
@@ -1488,7 +1488,7 @@ TEST(refreshGetMeta, normal2notexist) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
   taosMemoryFreeClear(tableMeta);
 
-  while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
+  while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
     taosMsleep(50);
   }
 
@@ -1541,7 +1541,7 @@ TEST(refreshGetMeta, normal2child) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n > 0) {
       break;
     }
@@ -1562,7 +1562,7 @@ TEST(refreshGetMeta, normal2child) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
   taosMemoryFreeClear(tableMeta);
 
-  while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
+  while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
     taosMsleep(50);
   }
 
@@ -1625,7 +1625,7 @@ TEST(refreshGetMeta, stable2child) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n > 0) {
       break;
     }
@@ -1647,7 +1647,7 @@ TEST(refreshGetMeta, stable2child) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
   taosMemoryFreeClear(tableMeta);
 
-  while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
+  while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
     taosMsleep(50);
   }
 
@@ -1710,7 +1710,7 @@ TEST(refreshGetMeta, stable2stable) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n > 0) {
       break;
     }
@@ -1732,7 +1732,7 @@ TEST(refreshGetMeta, stable2stable) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
   taosMemoryFreeClear(tableMeta);
 
-  while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
+  while (0 == ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
     taosMsleep(50);
   }
 
@@ -1798,7 +1798,7 @@ TEST(refreshGetMeta, child2stable) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n > 0) {
       break;
     }
@@ -1818,7 +1818,7 @@ TEST(refreshGetMeta, child2stable) {
   ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
   taosMemoryFreeClear(tableMeta);
 
-  while (2 != ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
+  while (2 != ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM)) {
     taosMsleep(50);
   }
 
@@ -2015,7 +2015,7 @@ TEST(dbVgroup, getSetDbVgroupCase) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n > 0) {
       break;
     }
@@ -2041,7 +2041,7 @@ TEST(dbVgroup, getSetDbVgroupCase) {
 
   while (true) {
     uint64_t n = 0;
-    ctgDbgGetStatNum("runtime.qDoneNum", (void *)&n);
+    ctgdGetStatNum("runtime.qDoneNum", (void *)&n);
     if (n != 3) {
       taosMsleep(50);
     } else {
@@ -2266,7 +2266,7 @@ TEST(rentTest, allRent) {
     ASSERT_EQ(tableMeta->tableInfo.precision, 1);
     ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
 
-    while (ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) {
+    while (ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM) < i) {
       taosMsleep(50);
     }
 

source/libs/command/CMakeLists.txt

+aux_source_directory(src COMMAND_SRC)
+add_library(command STATIC ${COMMAND_SRC})
+target_include_directories(
+        command
+        PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/command"
+        PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+)
+
+target_link_libraries(
+        command
+        PRIVATE os util nodes catalog function transport qcom
+)
+
+if(${BUILD_TEST})
+    ADD_SUBDIRECTORY(test)
+endif(${BUILD_TEST})
\ No newline at end of file

source/libs/command/inc/commandInt.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_QUERY_INT_H_
+#define _TD_QUERY_INT_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "nodes.h"
+#include "plannodes.h"
+#include "ttime.h"
+
+#define EXPLAIN_MAX_GROUP_NUM 100
+
+//newline area
+#define EXPLAIN_TAG_SCAN_FORMAT "Tag Scan on %s columns=%d width=%d"
+#define EXPLAIN_TBL_SCAN_FORMAT "Table Scan on %s columns=%d width=%d"
+#define EXPLAIN_SYSTBL_SCAN_FORMAT "System Table Scan on %s columns=%d width=%d"
+#define EXPLAIN_PROJECTION_FORMAT "Projection columns=%d width=%d"
+#define EXPLAIN_JOIN_FORMAT "%s between %d tables width=%d"
+#define EXPLAIN_AGG_FORMAT "Aggragate functions=%d"
+#define EXPLAIN_EXCHANGE_FORMAT "Data Exchange %d:1 width=%d"
+#define EXPLAIN_SORT_FORMAT "Sort on %d Column(s) width=%d"
+#define EXPLAIN_INTERVAL_FORMAT "Interval on Column %s functions=%d interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c width=%d"
+#define EXPLAIN_SESSION_FORMAT "Session gap=%" PRId64 " functions=%d width=%d"
+#define EXPLAIN_ORDER_FORMAT "Order: %s"
+#define EXPLAIN_FILTER_FORMAT "Filter: "
+#define EXPLAIN_FILL_FORMAT "Fill: %s"
+#define EXPLAIN_ON_CONDITIONS_FORMAT "Join Cond: "
+#define EXPLAIN_TIMERANGE_FORMAT "Time Range: [%" PRId64 ", %" PRId64 "]"
+
+//append area
+#define EXPLAIN_GROUPS_FORMAT " groups=%d"
+#define EXPLAIN_WIDTH_FORMAT " width=%d"
+#define EXPLAIN_LOOPS_FORMAT " loops=%d"
+#define EXPLAIN_REVERSE_FORMAT " reverse=%d"
+
+typedef struct SExplainGroup {
+  int32_t   nodeNum;
+  SSubplan *plan;
+  void     *execInfo;  //TODO
+} SExplainGroup;
+
+typedef struct SExplainResNode {
+  SNodeList*  pChildren;
+  SPhysiNode* pNode;
+  void*       pExecInfo;
+} SExplainResNode;
+
+typedef struct SQueryExplainRowInfo {
+  int32_t level;
+  int32_t len;
+  char   *buf;
+} SQueryExplainRowInfo;
+
+typedef struct SExplainCtx {
+  int32_t   totalSize;
+  bool      verbose;
+  char     *tbuf;
+  SArray   *rows;
+  SHashObj *groupHash;
+} SExplainCtx;
+
+#define EXPLAIN_ORDER_STRING(_order) ((TSDB_ORDER_ASC == _order) ? "Ascending" : "Descending")
+#define EXPLAIN_JOIN_STRING(_type) ((JOIN_TYPE_INNER == _type) ? "Inner join" : "Join")
+
+#define INVERAL_TIME_FROM_PRECISION_TO_UNIT(_t, _u, _p) (((_u) == 'n' || (_u) == 'y') ? (_t) : (convertTimeFromPrecisionToUnit(_t, _p, _u)))
+
+#define EXPLAIN_ROW_NEW(level, ...)                                                                               \
+  do {                                                                                                            \
+    if (isVerboseLine) {                                                                                          \
+      tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s", (level) * 2 + 3, "");       \
+    } else {                                                                                                      \
+      tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s%s", (level) * 2, "", "-> ");  \
+    }                                                                                                             \
+    tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__);         \
+  } while (0)
+  
+#define EXPLAIN_ROW_APPEND(...) tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__)
+#define EXPLAIN_ROW_END() do { varDataSetLen(tbuf, tlen); tlen += VARSTR_HEADER_SIZE; isVerboseLine = true; } while (0)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_QUERY_INT_H_*/

source/libs/command/src/command.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "command.h"
+#include "tdatablock.h"
+
+// #define SET_VARSTR(pData, val, pOffset)
+
+static void buildRspData(const STableMeta* pMeta, char* pData) {
+  int32_t* pColSizes = (int32_t*)pData;
+  pData += DESCRIBE_RESULT_COLS * sizeof(int32_t);
+  int32_t numOfRows = TABLE_TOTAL_COL_NUM(pMeta);
+
+  // Field
+  int32_t* pOffset = (int32_t*)pData;
+  pData += numOfRows * sizeof(int32_t);
+  for (int32_t i = 0; i < numOfRows; ++i) {
+    STR_TO_VARSTR(pData, pMeta->schema[i].name);
+    int16_t len = varDataTLen(pData);
+    pData += len;
+    *pOffset = pColSizes[0];
+    pOffset += 1;
+    pColSizes[0] += len;
+  }
+  
+  // Type
+  pOffset = (int32_t*)pData;
+  pData += numOfRows * sizeof(int32_t);
+  for (int32_t i = 0; i < numOfRows; ++i) {
+    STR_TO_VARSTR(pData, tDataTypes[pMeta->schema[i].type].name);
+    int16_t len = varDataTLen(pData);
+    pData += len;
+    *pOffset = pColSizes[1];
+    pOffset += 1;
+    pColSizes[1] += len;
+  }
+
+  // Length
+  pData += BitmapLen(numOfRows);
+  for (int32_t i = 0; i < numOfRows; ++i) {
+    *(int32_t*)pData = pMeta->schema[i].bytes;
+    pData += sizeof(int32_t);
+  }
+  pColSizes[2] = sizeof(int32_t) * numOfRows;
+
+  // Note
+  pOffset = (int32_t*)pData;
+  pData += numOfRows * sizeof(int32_t);
+  for (int32_t i = 0; i < numOfRows; ++i) {
+    STR_TO_VARSTR(pData, i >= pMeta->tableInfo.numOfColumns ? "TAG" : "");
+    int16_t len = varDataTLen(pData);
+    pData += len;
+    *pOffset = pColSizes[3];
+    pOffset += 1;
+    pColSizes[3] += len;
+  }
+
+  for (int32_t i = 0; i < DESCRIBE_RESULT_COLS; ++i) {
+    pColSizes[i] = htonl(pColSizes[i]);
+  }
+}
+
+static int32_t calcRspSize(const STableMeta* pMeta) {
+  int32_t numOfRows = TABLE_TOTAL_COL_NUM(pMeta);
+  return sizeof(SRetrieveTableRsp) + 
+      (numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_FIELD_LEN) + 
+      (numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_TYPE_LEN) +
+      (BitmapLen(numOfRows) + numOfRows * sizeof(int32_t)) + 
+      (numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_NOTE_LEN);
+}
+
+static int32_t execDescribe(SNode* pStmt, SRetrieveTableRsp** pRsp) {
+  SDescribeStmt* pDesc = (SDescribeStmt*)pStmt;
+  *pRsp = taosMemoryCalloc(1, calcRspSize(pDesc->pMeta));
+  if (NULL == *pRsp) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  (*pRsp)->useconds = 0;
+  (*pRsp)->completed = 1;
+  (*pRsp)->precision = 0;
+  (*pRsp)->compressed = 0;
+  (*pRsp)->compLen = 0;
+  (*pRsp)->numOfRows = htonl(TABLE_TOTAL_COL_NUM(pDesc->pMeta));
+  buildRspData(pDesc->pMeta, (*pRsp)->data);
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t execResetQueryCache() {
+  // todo
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t qExecCommand(SNode* pStmt, SRetrieveTableRsp** pRsp) {
+  switch (nodeType(pStmt)) {
+    case QUERY_NODE_DESCRIBE_STMT:
+      return execDescribe(pStmt, pRsp);
+    case QUERY_NODE_RESET_QUERY_CACHE_STMT:
+      return execResetQueryCache();
+    default:
+      break;
+  }
+  return TSDB_CODE_FAILED;
+}

source/libs/command/test/CMakeLists.txt

+MESSAGE(STATUS "build command unit test")
+
+# GoogleTest requires at least C++11
+SET(CMAKE_CXX_STANDARD 11)
+AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+
+ADD_EXECUTABLE(commandTest ${SOURCE_LIST})
+
+TARGET_INCLUDE_DIRECTORIES(
+        commandTest
+        PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/command/"
+        PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/command/inc"
+)
+
+TARGET_LINK_LIBRARIES(
+        commandTest
+        PUBLIC os util common nodes parser catalog transport gtest function qcom
+)
\ No newline at end of file

source/libs/command/test/commandTest.cpp

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <gtest/gtest.h>
+
+int main(int argc, char* argv[]) {
+	testing::InitGoogleTest(&argc, argv);
+	return RUN_ALL_TESTS();
+}

source/libs/executor/inc/executil.h

@@ -73,27 +73,12 @@ typedef struct SResultRowPosition {
 } SResultRowPosition;
 
 typedef struct SResultRowInfo {
-  SList       *pRows;
   SResultRowPosition *pPosition;
-  SResultRow **pResult;    // result list
   int32_t      size;       // number of result set
   int32_t      capacity;   // max capacity
   int32_t      curPos;     // current active result row index of pResult list
 } SResultRowInfo;
 
-typedef struct SResultRowPool {
-  int32_t elemSize;
-  int32_t blockSize;
-  int32_t numOfElemPerBlock;
-
-  struct {
-    int32_t blockIndex;
-    int32_t pos;
-  } position;
-
-  SArray* pData;    // SArray<void*>
-} SResultRowPool;
-
 struct STaskAttr;
 struct STaskRuntimeEnv;
 struct SUdfInfo;
@@ -109,25 +94,33 @@ void    resetResultRowInfo(struct STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo*
 int32_t numOfClosedResultRows(SResultRowInfo* pResultRowInfo);
 void    closeAllResultRows(SResultRowInfo* pResultRowInfo);
 
-int32_t initResultRow(SResultRow *pResultRow);
-void    closeResultRow(SResultRowInfo* pResultRowInfo, int32_t slot);
-bool    isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot);
-void    clearResultRow(struct STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow);
+void    initResultRow(SResultRow *pResultRow);
+void    closeResultRow(SResultRow* pResultRow);
+bool    isResultRowClosed(SResultRow* pResultRow);
 
 struct SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset);
 
-void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr);
 int32_t getRowNumForMultioutput(struct STaskAttr* pQueryAttr, bool topBottomQuery, bool stable);
 
-static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) {
-  assert(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size);
-  return pResultRowInfo->pResult[slot];
+static FORCE_INLINE SResultRow *getResultRow(SDiskbasedBuf* pBuf, SResultRowInfo *pResultRowInfo, int32_t slot) {
+  ASSERT(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size);
+  SResultRowPosition* pos = &pResultRowInfo->pPosition[slot];
+
+  SFilePage*  bufPage = (SFilePage*) getBufPage(pBuf, pos->pageId);
+  SResultRow* pRow = (SResultRow*)((char*)bufPage + pos->offset);
+  return pRow;
+}
+
+static FORCE_INLINE SResultRow *getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos) {
+  SFilePage*  bufPage = (SFilePage*) getBufPage(pBuf, pos->pageId);
+  SResultRow* pRow = (SResultRow*)((char*)bufPage + pos->offset);
+  return pRow;
 }
 
 static FORCE_INLINE char* getPosInResultPage(struct STaskAttr* pQueryAttr, SFilePage* page, int32_t rowOffset,
                                              int32_t offset) {
   assert(rowOffset >= 0 && pQueryAttr != NULL);
-
+  ASSERT(0);
 //  int32_t numOfRows = (int32_t)getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery);
 //  return ((char *)page->data) + rowOffset + offset * numOfRows;
 }
@@ -139,23 +132,14 @@ static FORCE_INLINE char* getPosInResultPage_rv(SFilePage* page, int32_t rowOffs
   return (char*) page + rowOffset + offset * numOfRows;
 }
 
-//bool isNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type);
-//bool notNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type);
-
-__filter_func_t getFilterOperator(int32_t lowerOptr, int32_t upperOptr);
-
-SResultRow* getNewResultRow(SResultRowPool* p);
-
 typedef struct {
   SArray* pResult;     // SArray<SResPair>
   int32_t colId;
 } SStddevInterResult;
 
-void interResToBinary(SBufferWriter* bw, SArray* pRes, int32_t tagLen);
-SArray* interResFromBinary(const char* data, int32_t len);
-void freeInterResult(void* param);
-
 void    initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo);
+void    initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList);
+
 void    cleanupGroupResInfo(SGroupResInfo* pGroupResInfo);
 bool    hasRemainDataInCurrentGroup(SGroupResInfo* pGroupResInfo);
 bool    hasRemainData(SGroupResInfo* pGroupResInfo);

source/libs/executor/inc/executorimpl.h

@@ -69,7 +69,7 @@ enum {
 
 typedef struct SResultRowCell {
   uint64_t    groupId;
-  SResultRow* pRow;
+  SResultRowPosition pos;
 } SResultRowCell;
 
 /**
@@ -277,8 +277,6 @@ typedef struct STaskRuntimeEnv {
   char*           keyBuf;               // window key buffer
   // The window result objects pool, all the resultRow Objects are allocated and managed by this object.
   char**          prevRow;
-  SResultRowPool* pool;
-
   SArray*         prevResult;  // intermediate result, SArray<SInterResult>
   STSBuf*         pTsBuf;      // timestamp filter list
   STSCursor       cur;
@@ -364,6 +362,7 @@ typedef struct SSourceDataInfo {
   int32_t               index;
   SRetrieveTableRsp    *pRsp;
   uint64_t              totalRows;
+  int32_t               code;
   EX_SOURCE_STATUS      status;
 } SSourceDataInfo;
 
@@ -422,6 +421,7 @@ typedef struct SStreamBlockScanInfo {
   uint64_t     numOfRows;     // total scanned rows
   uint64_t     numOfExec;     // execution times
   void*        readerHandle;  // stream block reader handle
+  SArray*      pColMatchInfo; //
 } SStreamBlockScanInfo;
 
 typedef struct SSysTableScanInfo {

source/libs/executor/src/executil.c

@@ -55,13 +55,11 @@ int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size) {
   pResultRowInfo->size     = 0;
   pResultRowInfo->curPos   = -1;
   pResultRowInfo->capacity = size;
-
-  pResultRowInfo->pResult = taosMemoryCalloc(pResultRowInfo->capacity, POINTER_BYTES);
   pResultRowInfo->pPosition = taosMemoryCalloc(pResultRowInfo->capacity, sizeof(SResultRowPosition));
-  if (pResultRowInfo->pResult == NULL || pResultRowInfo->pPosition == NULL) {
+
+  if (pResultRowInfo->pPosition == NULL) {
     return TSDB_CODE_QRY_OUT_OF_MEMORY;
   }
-
   return TSDB_CODE_SUCCESS;
 }
 
@@ -71,17 +69,17 @@ void cleanupResultRowInfo(SResultRowInfo *pResultRowInfo) {
   }
 
   if (pResultRowInfo->capacity == 0) {
-    assert(pResultRowInfo->pResult == NULL);
+//    assert(pResultRowInfo->pResult == NULL);
     return;
   }
 
   for(int32_t i = 0; i < pResultRowInfo->size; ++i) {
-    if (pResultRowInfo->pResult[i]) {
-      taosMemoryFreeClear(pResultRowInfo->pResult[i]->key);
-    }
+//    if (pResultRowInfo->pResult[i]) {
+//      taosMemoryFreeClear(pResultRowInfo->pResult[i]->key);
+//    }
   }
   
-  taosMemoryFreeClear(pResultRowInfo->pResult);
+  taosMemoryFreeClear(pResultRowInfo->pPosition);
 }
 
 void resetResultRowInfo(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo) {
@@ -90,8 +88,8 @@ void resetResultRowInfo(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRow
   }
 
   for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
-    SResultRow *pWindowRes = pResultRowInfo->pResult[i];
-    clearResultRow(pRuntimeEnv, pWindowRes);
+//    SResultRow *pWindowRes = pResultRowInfo->pResult[i];
+//    clearResultRow(pRuntimeEnv, pWindowRes);
 
     int32_t groupIndex = 0;
     int64_t uid = 0;
@@ -101,14 +99,13 @@ void resetResultRowInfo(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRow
   }
 
   pResultRowInfo->size     = 0;
-  pResultRowInfo->curPos  = -1;
 }
 
 int32_t numOfClosedResultRows(SResultRowInfo *pResultRowInfo) {
   int32_t i = 0;
-  while (i < pResultRowInfo->size && pResultRowInfo->pResult[i]->closed) {
-    ++i;
-  }
+//  while (i < pResultRowInfo->size && pResultRowInfo->pResult[i]->closed) {
+//    ++i;
+//  }
   
   return i;
 }
@@ -117,21 +114,22 @@ void closeAllResultRows(SResultRowInfo *pResultRowInfo) {
   assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size);
   
   for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
-    SResultRow* pRow = pResultRowInfo->pResult[i];
-    if (pRow->closed) {
-      continue;
-    }
+//    ASSERT(0);
+//    SResultRow* pRow = pResultRowInfo->pResult[i];
+//    if (pRow->closed) {
+//      continue;
+//    }
     
-    pRow->closed = true;
+//    pRow->closed = true;
   }
 }
 
-bool isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot) {
-  return (getResultRow(pResultRowInfo, slot)->closed == true);
+bool isResultRowClosed(SResultRow* pRow) {
+  return (pRow->closed == true);
 }
 
-void closeResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) {
-  getResultRow(pResultRowInfo, slot)->closed = true;
+void closeResultRow(SResultRow* pResultRow) {
+  pResultRow->closed = true;
 }
 
 void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow) {
@@ -181,29 +179,6 @@ size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
   return rowSize;
 }
 
-SResultRow* getNewResultRow(SResultRowPool* p) {
-  if (p == NULL) {
-    return NULL;
-  }
-
-  void* ptr = NULL;
-  if (p->position.pos == 0) {
-    ptr = taosMemoryCalloc(1, p->blockSize);
-    taosArrayPush(p->pData, &ptr);
-
-  } else {
-    size_t last = taosArrayGetSize(p->pData);
-
-    void** pBlock = taosArrayGet(p->pData, last - 1);
-    ptr = ((char*) (*pBlock)) + p->elemSize * p->position.pos;
-  }
-
-  p->position.pos = (p->position.pos + 1)%p->numOfElemPerBlock;
-  initResultRow(ptr);
-
-  return ptr;
-}
-
 void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) {
   assert(pGroupResInfo != NULL);
 
@@ -222,6 +197,16 @@ void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo)
   assert(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo));
 }
 
+void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList) {
+  if (pGroupResInfo->pRows != NULL) {
+    taosArrayDestroy(pGroupResInfo->pRows);
+  }
+
+  pGroupResInfo->pRows = pArrayList;
+  pGroupResInfo->index = 0;
+  ASSERT(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo));
+}
+
 bool hasRemainDataInCurrentGroup(SGroupResInfo* pGroupResInfo) {
   if (pGroupResInfo->pRows == NULL) {
     return false;
@@ -251,8 +236,9 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) {
   return (int32_t) taosArrayGetSize(pGroupResInfo->pRows);
 }
 
-static int64_t getNumOfResultWindowRes(STaskRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow, int32_t* rowCellInfoOffset) {
+static int64_t getNumOfResultWindowRes(STaskRuntimeEnv* pRuntimeEnv, SResultRowPosition *pos, int32_t* rowCellInfoOffset) {
   STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
+  ASSERT(0);
 
   for (int32_t j = 0; j < pQueryAttr->numOfOutput; ++j) {
     int32_t functionId = 0;//pQueryAttr->pExpr1[j].base.functionId;
@@ -295,25 +281,26 @@ static int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *
     return -1;
   }
 
+  ASSERT(0);
   STableQueryInfo** pList = supporter->pTableQueryInfo;
-  SResultRow* pWindowRes1 = pList[left]->resInfo.pResult[leftPos];
+//  SResultRow* pWindowRes1 = pList[left]->resInfo.pResult[leftPos];
 //  SResultRow * pWindowRes1 = getResultRow(&(pList[left]->resInfo), leftPos);
-  TSKEY leftTimestamp = pWindowRes1->win.skey;
+//  TSKEY leftTimestamp = pWindowRes1->win.skey;
 
 //  SResultRowInfo *pWindowResInfo2 = &(pList[right]->resInfo);
 //  SResultRow * pWindowRes2 = getResultRow(pWindowResInfo2, rightPos);
-  SResultRow* pWindowRes2 = pList[right]->resInfo.pResult[rightPos];
-  TSKEY rightTimestamp = pWindowRes2->win.skey;
+//  SResultRow* pWindowRes2 = pList[right]->resInfo.pResult[rightPos];
+//  TSKEY rightTimestamp = pWindowRes2->win.skey;
 
-  if (leftTimestamp == rightTimestamp) {
+//  if (leftTimestamp == rightTimestamp) {
     return 0;
-  }
+//  }
 
-  if (supporter->order == TSDB_ORDER_ASC) {
-    return (leftTimestamp > rightTimestamp)? 1:-1;
-  } else {
-    return (leftTimestamp < rightTimestamp)? 1:-1;
-  }
+//  if (supporter->order == TSDB_ORDER_ASC) {
+//    return (leftTimestamp > rightTimestamp)? 1:-1;
+//  } else {
+//    return (leftTimestamp < rightTimestamp)? 1:-1;
+//  }
 }
 
 int32_t tsAscOrder(const void* p1, const void* p2) {
@@ -321,11 +308,12 @@ int32_t tsAscOrder(const void* p1, const void* p2) {
   SResultRowCell* pc2 = (SResultRowCell*) p2;
 
   if (pc1->groupId == pc2->groupId) {
-    if (pc1->pRow->win.skey == pc2->pRow->win.skey) {
-      return 0;
-    } else {
-      return (pc1->pRow->win.skey < pc2->pRow->win.skey)? -1:1;
-    }
+    ASSERT(0);
+//    if (pc1->pRow->win.skey == pc2->pRow->win.skey) {
+//      return 0;
+//    } else {
+//      return (pc1->pRow->win.skey < pc2->pRow->win.skey)? -1:1;
+//    }
   } else {
     return (pc1->groupId < pc2->groupId)? -1:1;
   }
@@ -336,11 +324,12 @@ int32_t tsDescOrder(const void* p1, const void* p2) {
   SResultRowCell* pc2 = (SResultRowCell*) p2;
 
   if (pc1->groupId == pc2->groupId) {
-    if (pc1->pRow->win.skey == pc2->pRow->win.skey) {
-      return 0;
-    } else {
-      return (pc1->pRow->win.skey < pc2->pRow->win.skey)? 1:-1;
-    }
+    ASSERT(0);
+//    if (pc1->pRow->win.skey == pc2->pRow->win.skey) {
+//      return 0;
+//    } else {
+//      return (pc1->pRow->win.skey < pc2->pRow->win.skey)? 1:-1;
+//    }
   } else {
     return (pc1->groupId < pc2->groupId)? -1:1;
   }
@@ -374,13 +363,13 @@ static int32_t mergeIntoGroupResultImplRv(STaskRuntimeEnv *pRuntimeEnv, SGroupRe
       break;
     }
 
-    int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pResultRowCell->pRow, rowCellInfoOffset);
+    int64_t num = getNumOfResultWindowRes(pRuntimeEnv, &pResultRowCell->pos, rowCellInfoOffset);
     if (num <= 0) {
       continue;
     }
 
-    taosArrayPush(pGroupResInfo->pRows, &pResultRowCell->pRow);
-    pResultRowCell->pRow->numOfRows = (uint32_t) num;
+    taosArrayPush(pGroupResInfo->pRows, &pResultRowCell->pos);
+//    pResultRowCell->pRow->numOfRows = (uint32_t) num;
   }
 
   return TSDB_CODE_SUCCESS;
@@ -439,9 +428,10 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv
     int32_t tableIndex = tMergeTreeGetChosenIndex(pTree);
 
     SResultRowInfo *pWindowResInfo = &pTableQueryInfoList[tableIndex]->resInfo;
-    SResultRow  *pWindowRes = getResultRow(pWindowResInfo, cs.rowIndex[tableIndex]);
+    ASSERT(0);
+    SResultRow  *pWindowRes = NULL;//getResultRow(pBuf, pWindowResInfo, cs.rowIndex[tableIndex]);
 
-    int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes, rowCellInfoOffset);
+    int64_t num = 0;//getNumOfResultWindowRes(pRuntimeEnv, pWindowRes, rowCellInfoOffset);
     if (num <= 0) {
       cs.rowIndex[tableIndex] += 1;
 

source/libs/index/inc/indexSparse.h → source/libs/index/inc/indexFstSparse.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_INDEX_SPARSE_H_
-#define _TD_INDEX_SPARSE_H_
+#ifndef _TD_INDEX_FST_SPARSE_H_
+#define _TD_INDEX_FST_SPARSE_H_
 
 #include "tarray.h"
 
@@ -23,8 +23,8 @@ extern "C" {
 #endif
 
 typedef struct FstSparseSet {
-  SArray *dense;
-  SArray *sparse;
+  uint32_t *dense;
+  uint32_t *sparse;
   int32_t   size;
 } FstSparseSet;
 

source/libs/index/src/indexFstCommon.c

 /*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+YAML:9:25: error: unknown key 'AlignConsecutiveMacros' * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
  *
  * This program is free software: you can use, redistribute, and/or modify
  * it under the terms of the GNU Affero General Public License, version 3

source/libs/nodes/src/nodesTraverseFuncs.c

@@ -17,7 +17,8 @@
 
 typedef enum ETraversalOrder {
   TRAVERSAL_PREORDER = 1,
-  TRAVERSAL_POSTORDER
+  TRAVERSAL_INORDER,
+  TRAVERSAL_POSTORDER,
 } ETraversalOrder;
 
 static EDealRes walkList(SNodeList* pNodeList, ETraversalOrder order, FNodeWalker walker, void* pContext);