Merge branch '3.0' into test/3.0_math_functions

contrib/CMakeLists.txt

@@ -220,7 +220,7 @@ endif(${BUILD_WITH_NURAFT})
 if(${BUILD_PTHREAD})
     set(CMAKE_BUILD_TYPE release)
     add_definitions(-DPTW32_STATIC_LIB)
-    add_subdirectory(pthread EXCLUDE_FROM_ALL)
+    add_subdirectory(pthread)
     set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread)
     add_library(pthread STATIC IMPORTED GLOBAL)
     SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib)
@@ -228,7 +228,7 @@ endif()
 
 # iconv
 if(${BUILD_WITH_ICONV})
-    add_subdirectory(iconv EXCLUDE_FROM_ALL)
+    add_library(iconv STATIC iconv/win_iconv.c)
 endif(${BUILD_WITH_ICONV})
 
 # wingetopt

example/src/tstream.c

@@ -25,7 +25,7 @@ int32_t init_env() {
     return -1;
   }
 
-  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2");
+  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1");
   if (taos_errno(pRes) != 0) {
     printf("error in create db, reason:%s\n", taos_errstr(pRes));
     return -1;
@@ -78,11 +78,14 @@ int32_t create_stream() {
   taos_free_result(pRes);
 
   /*const char* sql = "select min(k), max(k), sum(k) from tu1";*/
-  const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";
+  /*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/
   /*const char* sql = "select sum(k) from tu1 interval(10m)";*/
-  pRes = tmq_create_stream(pConn, "stream1", "out1", sql);
+  /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/
+  pRes = taos_query(
+      pConn,
+      "create stream stream1 trigger window_close as select min(k), max(k), sum(k) as sum_of_k from tu1 interval(10m)");
   if (taos_errno(pRes) != 0) {
-    printf("failed to create stream out1, reason:%s\n", taos_errstr(pRes));
+    printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes));
     return -1;
   }
   taos_free_result(pRes);

include/client/taos.h

@@ -86,9 +86,9 @@ typedef struct taosField {
 } TAOS_FIELD;
 
 #ifdef WINDOWS
-  #define DLL_EXPORT    __declspec(dllexport)
+#define DLL_EXPORT __declspec(dllexport)
 #else
-  #define DLL_EXPORT 
+#define DLL_EXPORT
 #endif
 
 typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code);
@@ -121,6 +121,7 @@ typedef struct setConfRet {
 DLL_EXPORT void       taos_cleanup(void);
 DLL_EXPORT int        taos_options(TSDB_OPTION option, const void *arg, ...);
 DLL_EXPORT setConfRet taos_set_config(const char *config);
+DLL_EXPORT int        taos_init(void);
 DLL_EXPORT TAOS      *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
 DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen,
                                 const char *db, int dbLen, uint16_t port);
@@ -270,10 +271,8 @@ DLL_EXPORT int64_t     tmq_get_response_offset(tmq_message_t *message);
 /* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */
 #if 0
 DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen);
-#endif
-
 DLL_EXPORT TAOS_RES *tmq_create_stream(TAOS *taos, const char *streamName, const char *tbName, const char *sql);
-
+#endif
 /* ------------------------------ TMQ END -------------------------------- */
 #if 1  // Shuduo: temporary enable for app build
 typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code);

include/common/systable.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "os.h"
+
+#ifndef TDENGINE_SYSTABLE_H
+#define TDENGINE_SYSTABLE_H
+
+#define TSDB_INFORMATION_SCHEMA_DB            "information_schema"
+#define TSDB_PERFORMANCE_SCHEMA_DB            "performance_schema"
+#define TSDB_INS_TABLE_DNODES                 "dnodes"
+#define TSDB_INS_TABLE_MNODES                 "mnodes"
+#define TSDB_INS_TABLE_MODULES                "modules"
+#define TSDB_INS_TABLE_QNODES                 "qnodes"
+#define TSDB_INS_TABLE_BNODES                 "bnodes"
+#define TSDB_INS_TABLE_SNODES                 "snodes"
+#define TSDB_INS_TABLE_CLUSTER                "cluster"
+#define TSDB_INS_TABLE_USER_DATABASES         "user_databases"
+#define TSDB_INS_TABLE_USER_FUNCTIONS         "user_functions"
+#define TSDB_INS_TABLE_USER_INDEXES           "user_indexes"
+#define TSDB_INS_TABLE_USER_STABLES           "user_stables"
+#define TSDB_INS_TABLE_USER_STREAMS           "user_streams"
+#define TSDB_INS_TABLE_USER_TABLES            "user_tables"
+#define TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED "user_table_distributed"
+#define TSDB_INS_TABLE_USER_USERS             "user_users"
+#define TSDB_INS_TABLE_LICENCES               "grants"
+#define TSDB_INS_TABLE_VGROUPS                "vgroups"
+#define TSDB_INS_TABLE_VNODES                 "vnodes"
+#define TSDB_INS_TABLE_CONFIGS                "configs"
+
+#define TSDB_PERFORMANCE_SCHEMA_DB            "performance_schema"
+#define TSDB_PERFS_TABLE_SMAS                 "smas"
+#define TSDB_PERFS_TABLE_SUBSCRIBES           "subscribes"
+#define TSDB_PERFS_TABLE_CONNECTIONS          "connections"
+#define TSDB_PERFS_TABLE_QUERIES              "queries"
+#define TSDB_PERFS_TABLE_TOPICS               "topics"
+#define TSDB_PERFS_TABLE_CONSUMERS            "consumers"
+#define TSDB_PERFS_TABLE_SUBSCRIPTIONS        "subscriptions"
+#define TSDB_PERFS_TABLE_OFFSETS              "offsets"
+#define TSDB_PERFS_TABLE_TRANS                "trans"
+#define TSDB_PERFS_TABLE_STREAMS              "streams"
+
+typedef struct SSysDbTableSchema {
+  const char   *name;
+  const int32_t type;
+  const int32_t bytes;
+} SSysDbTableSchema;
+
+typedef struct SSysTableMeta {
+  const char              *name;
+  const SSysDbTableSchema *schema;
+  const int32_t            colNum;
+} SSysTableMeta;
+
+void getInfosDbMeta(const SSysTableMeta** pInfosTableMeta, size_t* size);
+void getPerfDbMeta(const SSysTableMeta** pPerfsTableMeta, size_t* size);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_SYSTABLE_H

include/common/taosdef.h

@@ -82,6 +82,8 @@ extern char *qtypeStr[];
 
 #define TSDB_PORT_HTTP      11
 
+#undef TD_DEBUG_PRINT_ROW
+
 #ifdef __cplusplus
 }
 #endif

include/common/tmsg.h

@@ -438,6 +438,7 @@ typedef struct {
 
 int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp);
 int32_t tDeserializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp);
+void    tFreeSGetUserAuthRsp(SGetUserAuthRsp* pRsp);
 
 typedef struct {
   int16_t colId;     // column id
@@ -530,27 +531,24 @@ int32_t tDeserializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pR
 typedef struct {
   char    db[TSDB_DB_FNAME_LEN];
   int32_t numOfVgroups;
-  int32_t cacheBlockSize;  // MB
-  int32_t totalBlocks;
+  int32_t numOfStables;  // single_stable
+  int32_t buffer;        // MB
+  int32_t pageSize;
+  int32_t pages;
   int32_t daysPerFile;
   int32_t daysToKeep0;
   int32_t daysToKeep1;
   int32_t daysToKeep2;
   int32_t minRows;
   int32_t maxRows;
-  int32_t commitTime;
   int32_t fsyncPeriod;
-  int32_t ttl;
   int8_t  walLevel;
   int8_t  precision;  // time resolution
   int8_t  compression;
   int8_t  replications;
   int8_t  strict;
-  int8_t  update;
   int8_t  cacheLastRow;
   int8_t  ignoreExist;
-  int8_t  streamMode;
-  int8_t  singleSTable;
   int32_t numOfRetensions;
   SArray* pRetensions;  // SRetention
 } SCreateDbReq;
@@ -561,7 +559,10 @@ void    tFreeSCreateDbReq(SCreateDbReq* pReq);
 
 typedef struct {
   char    db[TSDB_DB_FNAME_LEN];
-  int32_t totalBlocks;
+  int32_t buffer;
+  int32_t pageSize;
+  int32_t pages;
+  int32_t daysPerFile;
   int32_t daysToKeep0;
   int32_t daysToKeep1;
   int32_t daysToKeep2;
@@ -625,26 +626,23 @@ int32_t tDeserializeSDbCfgReq(void* buf, int32_t bufLen, SDbCfgReq* pReq);
 
 typedef struct {
   int32_t numOfVgroups;
-  int32_t cacheBlockSize;
-  int32_t totalBlocks;
+  int32_t numOfStables;
+  int32_t buffer;
+  int32_t pageSize;
+  int32_t pages;
   int32_t daysPerFile;
   int32_t daysToKeep0;
   int32_t daysToKeep1;
   int32_t daysToKeep2;
   int32_t minRows;
   int32_t maxRows;
-  int32_t commitTime;
   int32_t fsyncPeriod;
-  int32_t ttl;
   int8_t  walLevel;
   int8_t  precision;
   int8_t  compression;
   int8_t  replications;
   int8_t  strict;
-  int8_t  update;
   int8_t  cacheLastRow;
-  int8_t  streamMode;
-  int8_t  singleSTable;
   int32_t numOfRetensions;
   SArray* pRetensions;
 } SDbCfgRsp;
@@ -844,15 +842,16 @@ typedef struct {
   char     db[TSDB_DB_FNAME_LEN];
   int64_t  dbUid;
   int32_t  vgVersion;
-  int32_t  cacheBlockSize;
-  int32_t  totalBlocks;
+   int32_t numOfStables;
+  int32_t  buffer;
+  int32_t  pageSize;
+  int32_t  pages;
   int32_t  daysPerFile;
   int32_t  daysToKeep0;
   int32_t  daysToKeep1;
   int32_t  daysToKeep2;
   int32_t  minRows;
   int32_t  maxRows;
-  int32_t  commitTime;
   int32_t  fsyncPeriod;
   uint32_t hashBegin;
   uint32_t hashEnd;
@@ -861,11 +860,9 @@ typedef struct {
   int8_t   precision;
   int8_t   compression;
   int8_t   strict;
-  int8_t   update;
   int8_t   cacheLastRow;
   int8_t   replica;
   int8_t   selfIndex;
-  int8_t   streamMode;
   SReplica replicas[TSDB_MAX_REPLICA];
   int32_t  numOfRetensions;
   SArray*  pRetensions;  // SRetention
@@ -895,10 +892,14 @@ int32_t tDeserializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq
 
 typedef struct {
   int32_t  vgVersion;
-  int32_t  totalBlocks;
+  int32_t  buffer;
+  int32_t  pageSize;
+  int32_t  pages;
+  int32_t  daysPerFile;
   int32_t  daysToKeep0;
   int32_t  daysToKeep1;
   int32_t  daysToKeep2;
+  int32_t  fsyncPeriod;
   int8_t   walLevel;
   int8_t   strict;
   int8_t   cacheLastRow;
@@ -953,7 +954,6 @@ typedef struct {
   int32_t  numOfColumns;
   int8_t   precision;
   int8_t   tableType;
-  int8_t   update;
   int32_t  sversion;
   int32_t  tversion;
   uint64_t suid;
@@ -1038,6 +1038,7 @@ typedef struct {
   int8_t  compressed;
   int32_t compLen;
   int32_t numOfRows;
+  int32_t numOfCols;
   char    data[];
 } SRetrieveTableRsp;
 
@@ -1280,8 +1281,9 @@ typedef struct {
 #define STREAM_TRIGGER_WINDOW_CLOSE 2
 
 typedef struct {
-  char    name[TSDB_TOPIC_FNAME_LEN];
-  char    outputSTbName[TSDB_TABLE_FNAME_LEN];
+  char    name[TSDB_TABLE_FNAME_LEN];
+  char    sourceDB[TSDB_DB_FNAME_LEN];
+  char    targetStbFullName[TSDB_TABLE_FNAME_LEN];
   int8_t  igExists;
   char*   sql;
   char*   ast;
@@ -1332,7 +1334,7 @@ int32_t tDeserializeSCMCreateTopicRsp(void* buf, int32_t bufLen, SCMCreateTopicR
 
 typedef struct {
   int64_t consumerId;
-} SMqConsumerLostMsg;
+} SMqConsumerLostMsg, SMqConsumerRecoverMsg;
 
 typedef struct {
   int64_t consumerId;
@@ -1472,7 +1474,6 @@ _err:
 // this message is sent from mnode to mnode(read thread to write thread), so there is no need for serialization or
 // deserialization
 typedef struct {
-  int8_t*   mqInReb;
   SHashObj* rebSubHash;  // SHashObj<key, SMqRebSubscribe>
 } SMqDoRebalanceMsg;
 
@@ -1520,8 +1521,8 @@ typedef struct {
   char*   qmsg2;  // pAst2:qmsg2:SRetention2 => trigger aggr task2
 } SRSmaParam;
 
-int tEncodeSRSmaParam(SCoder* pCoder, const SRSmaParam* pRSmaParam);
-int tDecodeSRSmaParam(SCoder* pCoder, SRSmaParam* pRSmaParam);
+int32_t tEncodeSRSmaParam(SCoder* pCoder, const SRSmaParam* pRSmaParam);
+int32_t tDecodeSRSmaParam(SCoder* pCoder, SRSmaParam* pRSmaParam);
 
 typedef struct SVCreateStbReq {
   const char*    name;
@@ -1537,6 +1538,10 @@ int tDecodeSVCreateStbReq(SCoder* pCoder, SVCreateStbReq* pReq);
 
 typedef struct SVDropStbReq {
   // data
+#ifdef WINDOWS
+  size_t avoidCompilationErrors;
+#endif
+
 } SVDropStbReq;
 
 typedef struct SVCreateStbRsp {
@@ -2116,7 +2121,7 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SCoder* pDecoder, SSchemaWrapp
   if (tDecodeI32v(pDecoder, &pSW->nCols) < 0) return -1;
   if (tDecodeI32v(pDecoder, &pSW->sver) < 0) return -1;
 
-  pSW->pSchema = (SSchema*)TCODER_MALLOC(pDecoder, sizeof(SSchema) * pSW->nCols);
+  pSW->pSchema = (SSchema*)tCoderMalloc(pDecoder, sizeof(SSchema) * pSW->nCols);
   if (pSW->pSchema == NULL) return -1;
   for (int32_t i = 0; i < pSW->nCols; i++) {
     if (tDecodeSSchema(pDecoder, &pSW->pSchema[i]) < 0) return -1;

include/common/tmsgdef.h

@@ -145,10 +145,11 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp)
-  TD_DEF_MSG_TYPE(TDMT_MND_MQ_ASK_EP, "mnode-mq-ask-ep", SMqAskEpReq, SMqAskEpReq)
-  TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq)
-  TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMTimerReq, SMTimerReq)
-  TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_ASK_EP, "mnode-mq-ask-ep", SMqAskEpReq, SMqAskEpRsp)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMqConsumerLostMsg, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_RECOVER, "mnode-mq-consumer-recover", SMqConsumerRecoverMsg, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "mnode-create-stream", SCMCreateStreamReq, SCMCreateStreamRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STREAM, "mnode-alter-stream", NULL, NULL)

include/common/trow.h

@@ -214,6 +214,16 @@ STSRow *tdRowDup(STSRow *row);
 static FORCE_INLINE SKvRowIdx *tdKvRowColIdxAt(STSRow *pRow, col_id_t idx) {
   return (SKvRowIdx *)TD_ROW_COL_IDX(pRow) + idx;
 }
+
+static FORCE_INLINE int16_t tdKvRowColIdAt(STSRow *pRow, col_id_t idx) {
+  ASSERT(idx >= 0);
+  if (idx == 0) {
+    return PRIMARYKEY_TIMESTAMP_COL_ID;
+  }
+
+  return ((SKvRowIdx *)TD_ROW_COL_IDX(pRow) + idx - 1)->colId;
+}
+
 static FORCE_INLINE void *tdKVRowColVal(STSRow *pRow, SKvRowIdx *pIdx) { return POINTER_SHIFT(pRow, pIdx->offset); }
 
 #define TD_ROW_OFFSET(p) ((p)->toffset);  // During ParseInsert when without STSchema, how to get the offset for STpRow?
@@ -585,6 +595,34 @@ static FORCE_INLINE int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols,
   return TSDB_CODE_SUCCESS;
 }
 
+/**
+ * @brief
+ *
+ * @param pBuilder
+ * @param nCols
+ * @param nBoundCols use -1 if not available
+ * @param flen
+ * @return FORCE_INLINE
+ */
+static FORCE_INLINE int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen) {
+  pBuilder->flen = flen;
+  pBuilder->nCols = nCols;
+  if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) {
+    TASSERT(0);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+#ifdef TD_SUPPORT_BITMAP
+  // the primary TS key is stored separatedly
+  pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1);
+#else
+  pBuilder->nBitmaps = 0;
+  pBuilder->nBoundBitmaps = 0;
+#endif
+  return TSDB_CODE_SUCCESS;
+}
+
+
 /**
  * @brief To judge row type: STpRow/SKvRow
  *
@@ -668,7 +706,7 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
     case TD_ROW_KV:
 #ifdef TD_SUPPORT_BITMAP
       pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols);
-      memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBitmaps);
+      memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBoundBitmaps);
 #endif
       len = TD_ROW_HEAD_LEN + TD_ROW_NCOLS_LEN + (pBuilder->nBoundCols - 1) * sizeof(SKvRowIdx) +
             pBuilder->nBoundBitmaps;  // add
@@ -1092,7 +1130,7 @@ static FORCE_INLINE bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId,
   STSRow    *pRow = pIter->pRow;
   SKvRowIdx *pKvIdx = NULL;
   bool       colFound = false;
-  col_id_t   kvNCols = tdRowGetNCols(pRow);
+  col_id_t   kvNCols = tdRowGetNCols(pRow) - 1;
   while (*nIdx < kvNCols) {
     pKvIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(pRow), *nIdx * sizeof(SKvRowIdx));
     if (pKvIdx->colId == colId) {
@@ -1108,7 +1146,14 @@ static FORCE_INLINE bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId,
     }
   }
 
-  if (!colFound) return false;
+  if (!colFound) {
+    if (colId <= pIter->maxColId) {
+      pVal->valType = TD_VTYPE_NONE;
+      return true;
+    } else {
+      return false;
+    }
+  }
 
 #ifdef TD_SUPPORT_BITMAP
   int16_t colIdx = -1;
@@ -1352,11 +1397,11 @@ static void tdSCellValPrint(SCellVal *pVal, int8_t colType) {
   }
 }
 
-static void tdSRowPrint(STSRow *row, STSchema *pSchema) {
+static void tdSRowPrint(STSRow *row, STSchema *pSchema, const char* tag) {
   STSRowIter iter = {0};
   tdSTSRowIterInit(&iter, pSchema);
   tdSTSRowIterReset(&iter, row);
-  printf(">>>");
+  printf("%s >>>", tag);
   for (int i = 0; i < pSchema->numOfCols; ++i) {
     STColumn *stCol = pSchema->columns + i;
     SCellVal  sVal = { 255, NULL};
@@ -1368,7 +1413,6 @@ static void tdSRowPrint(STSRow *row, STSchema *pSchema) {
   }
   printf("\n");
 }
-
 #ifdef TROW_ORIGIN_HZ
 typedef struct {
   uint32_t nRows;

include/common/ttokendef.h

@@ -67,101 +67,101 @@
 #define TK_IF            49
 #define TK_NOT           50
 #define TK_EXISTS        51
-#define TK_BLOCKS                          52
-#define TK_CACHE                           53
-#define TK_CACHELAST                       54
-#define TK_COMP                            55
-#define TK_DAYS                            56
-#define TK_NK_VARIABLE                     57
-#define TK_FSYNC                           58
-#define TK_MAXROWS                         59
-#define TK_MINROWS                         60
-#define TK_KEEP                            61
-#define TK_PRECISION                       62
-#define TK_QUORUM                          63
+#define TK_BUFFER        52
+#define TK_CACHELAST     53
+#define TK_COMP          54
+#define TK_DAYS          55
+#define TK_NK_VARIABLE   56
+#define TK_FSYNC         57
+#define TK_MAXROWS       58
+#define TK_MINROWS       59
+#define TK_KEEP          60
+#define TK_PAGES         61
+#define TK_PAGESIZE      62
+#define TK_PRECISION     63
 #define TK_REPLICA       64
-#define TK_TTL                             65
+#define TK_STRICT        65
 #define TK_WAL           66
 #define TK_VGROUPS       67
 #define TK_SINGLE_STABLE 68
-#define TK_STREAM_MODE                     69
-#define TK_RETENTIONS                      70
-#define TK_STRICT                          71
-#define TK_NK_COMMA                        72
-#define TK_NK_COLON                        73
-#define TK_TABLE                           74
-#define TK_NK_LP                           75
-#define TK_NK_RP                           76
-#define TK_STABLE                          77
-#define TK_ADD                             78
-#define TK_COLUMN                          79
-#define TK_MODIFY                          80
-#define TK_RENAME                          81
-#define TK_TAG                             82
-#define TK_SET                             83
-#define TK_NK_EQ                           84
-#define TK_USING                           85
-#define TK_TAGS                            86
-#define TK_NK_DOT                          87
-#define TK_COMMENT                         88
-#define TK_BOOL                            89
-#define TK_TINYINT                         90
-#define TK_SMALLINT                        91
-#define TK_INT                             92
-#define TK_INTEGER                         93
-#define TK_BIGINT                          94
-#define TK_FLOAT                           95
-#define TK_DOUBLE                          96
-#define TK_BINARY                          97
-#define TK_TIMESTAMP                       98
-#define TK_NCHAR                           99
-#define TK_UNSIGNED                       100
-#define TK_JSON                           101
-#define TK_VARCHAR                        102
-#define TK_MEDIUMBLOB                     103
-#define TK_BLOB                           104
-#define TK_VARBINARY                      105
-#define TK_DECIMAL                        106
-#define TK_SMA                            107
+#define TK_RETENTIONS    69
+#define TK_NK_COMMA      70
+#define TK_NK_COLON      71
+#define TK_TABLE         72
+#define TK_NK_LP         73
+#define TK_NK_RP         74
+#define TK_STABLE        75
+#define TK_ADD           76
+#define TK_COLUMN        77
+#define TK_MODIFY        78
+#define TK_RENAME        79
+#define TK_TAG           80
+#define TK_SET           81
+#define TK_NK_EQ         82
+#define TK_USING         83
+#define TK_TAGS          84
+#define TK_NK_DOT        85
+#define TK_COMMENT       86
+#define TK_BOOL          87
+#define TK_TINYINT       88
+#define TK_SMALLINT      89
+#define TK_INT           90
+#define TK_INTEGER       91
+#define TK_BIGINT        92
+#define TK_FLOAT         93
+#define TK_DOUBLE        94
+#define TK_BINARY        95
+#define TK_TIMESTAMP     96
+#define TK_NCHAR         97
+#define TK_UNSIGNED      98
+#define TK_JSON          99
+#define TK_VARCHAR       100
+#define TK_MEDIUMBLOB    101
+#define TK_BLOB          102
+#define TK_VARBINARY     103
+#define TK_DECIMAL       104
+#define TK_DELAY         105
+#define TK_FILE_FACTOR   106
+#define TK_NK_FLOAT      107
 #define TK_ROLLUP        108
-#define TK_FILE_FACTOR                    109
-#define TK_NK_FLOAT                       110
-#define TK_DELAY                          111
-#define TK_SHOW                           112
-#define TK_DATABASES                      113
-#define TK_TABLES                         114
-#define TK_STABLES                        115
-#define TK_MNODES                         116
-#define TK_MODULES                        117
-#define TK_QNODES                         118
-#define TK_FUNCTIONS                      119
-#define TK_INDEXES                        120
-#define TK_FROM                           121
-#define TK_ACCOUNTS                       122
-#define TK_APPS                           123
-#define TK_CONNECTIONS                    124
-#define TK_LICENCE                        125
-#define TK_GRANTS                         126
-#define TK_QUERIES                        127
-#define TK_SCORES                         128
-#define TK_TOPICS                         129
-#define TK_VARIABLES                      130
-#define TK_BNODES                         131
-#define TK_SNODES                         132
-#define TK_CLUSTER                        133
-#define TK_LIKE                           134
-#define TK_INDEX                          135
-#define TK_FULLTEXT                       136
-#define TK_FUNCTION                       137
-#define TK_INTERVAL                       138
-#define TK_TOPIC                          139
-#define TK_AS                             140
-#define TK_WITH                           141
-#define TK_SCHEMA                         142
-#define TK_DESC                           143
-#define TK_DESCRIBE                       144
-#define TK_RESET                          145
-#define TK_QUERY                          146
+#define TK_TTL           109
+#define TK_SMA           110
+#define TK_SHOW          111
+#define TK_DATABASES     112
+#define TK_TABLES        113
+#define TK_STABLES       114
+#define TK_MNODES        115
+#define TK_MODULES       116
+#define TK_QNODES        117
+#define TK_FUNCTIONS     118
+#define TK_INDEXES       119
+#define TK_FROM          120
+#define TK_ACCOUNTS      121
+#define TK_APPS          122
+#define TK_CONNECTIONS   123
+#define TK_LICENCE       124
+#define TK_GRANTS        125
+#define TK_QUERIES       126
+#define TK_SCORES        127
+#define TK_TOPICS        128
+#define TK_VARIABLES     129
+#define TK_BNODES        130
+#define TK_SNODES        131
+#define TK_CLUSTER       132
+#define TK_LIKE          133
+#define TK_INDEX         134
+#define TK_FULLTEXT      135
+#define TK_FUNCTION      136
+#define TK_INTERVAL      137
+#define TK_TOPIC         138
+#define TK_AS            139
+#define TK_WITH          140
+#define TK_SCHEMA        141
+#define TK_DESC          142
+#define TK_DESCRIBE      143
+#define TK_RESET         144
+#define TK_QUERY         145
+#define TK_CACHE         146
 #define TK_EXPLAIN       147
 #define TK_ANALYZE       148
 #define TK_VERBOSE       149

include/dnode/mnode/sdb/sdb.h

@@ -331,7 +331,6 @@ int32_t  sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver);
 int32_t  sdbGetRawTotalSize(SSdbRaw *pRaw);
 
 SSdbRow *sdbAllocRow(int32_t objSize);
-void     sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc);
 void    *sdbGetRowObj(SSdbRow *pRow);
 
 typedef struct SSdb {

include/libs/executor/dataSinkMgt.h

@@ -45,6 +45,7 @@ typedef struct SInputData {
 
 typedef struct SOutputData {
   int32_t numOfRows;
+  int32_t numOfCols;
   int8_t  compressed;
   char*   pData;
   bool    queryEnd;

include/libs/executor/executor.h

@@ -32,6 +32,8 @@ typedef struct SReadHandle {
   void* reader;
   void* meta;
   void* config;
+  void* vnode;
+  void* mnd;
 } SReadHandle;
 
 #define STREAM_DATA_TYPE_SUBMIT_BLOCK 0x1

include/libs/function/function.h

@@ -206,6 +206,8 @@ typedef struct SqlFunctionCtx {
   struct SDiskbasedBuf  *pBuf;
   struct SSDataBlock    *pSrcBlock;
   int32_t                curBufPage;
+
+  char                   udfName[TSDB_FUNC_NAME_LEN];
 } SqlFunctionCtx;
 
 enum {
@@ -334,8 +336,6 @@ int32_t udfcOpen();
  */
 int32_t udfcClose();
 
-typedef void *UdfcFuncHandle;
-
 #ifdef __cplusplus
 }
 #endif

include/libs/function/functionMgt.h

@@ -40,6 +40,7 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_STDDEV,
   FUNCTION_TYPE_SUM,
   FUNCTION_TYPE_TWA,
+  FUNCTION_TYPE_HISTOGRAM,
 
   // nonstandard SQL function
   FUNCTION_TYPE_BOTTOM = 500,
@@ -162,6 +163,7 @@ EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWin
 
 int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet);
 int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet);
+int32_t fmGetUdafExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet);
 
 #ifdef __cplusplus
 }

source/libs/function/inc/tudf.h → include/libs/function/tudf.h

@@ -42,8 +42,7 @@ enum {
   UDFC_CODE_INVALID_STATE = -5
 };
 
-
-
+typedef void *UdfcFuncHandle;
 
 /**
  * setup udf
@@ -95,6 +94,7 @@ typedef struct SUdfDataBlock {
 typedef struct SUdfInterBuf {
   int32_t bufLen;
   char* buf;
+  int8_t numOfResult; //zero or one
 } SUdfInterBuf;
 
 // output: interBuf
@@ -118,6 +118,10 @@ int32_t callUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t nu
  */
 int32_t teardownUdf(UdfcFuncHandle handle);
 
+bool udfAggGetEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
+bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
+int32_t udfAggProcess(struct SqlFunctionCtx *pCtx);
+int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock);
 // end API to taosd and qworker
 //=============================================================================================================================
 // begin API to UDF writer.
@@ -133,11 +137,11 @@ typedef int32_t (*TUdfTeardownFunc)();
 //typedef int32_t addVariableLengthColumnData(SColumnData *columnData, int rowIndex, bool isNull, int32_t dataLen, char * data);
 
 typedef int32_t (*TUdfFreeUdfColumnFunc)(SUdfColumn* column);
+typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock* block, SUdfColumn *resultCol);
 
-typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock block, SUdfColumn *resultCol);
-typedef int32_t (*TUdfAggInitFunc)(SUdfInterBuf *buf);
-typedef int32_t (*TUdfAggProcessFunc)(SUdfDataBlock block, SUdfInterBuf *interBuf);
-typedef int32_t (*TUdfAggFinalizeFunc)(SUdfInterBuf buf, SUdfInterBuf *resultData);
+typedef int32_t (*TUdfAggStartFunc)(SUdfInterBuf *buf);
+typedef int32_t (*TUdfAggProcessFunc)(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf);
+typedef int32_t (*TUdfAggFinishFunc)(SUdfInterBuf* buf, SUdfInterBuf *resultData);
 
 
 // end API to UDF writer

include/libs/nodes/cmdnodes.h

@@ -30,24 +30,25 @@ extern "C" {
 
 typedef struct SDatabaseOptions {
   ENodeType   type;
-  SValueNode* pNumOfBlocks;
-  SValueNode* pCacheBlockSize;
-  SValueNode* pCachelast;
-  SValueNode* pCompressionLevel;
+  int32_t     buffer;
+  int8_t      cachelast;
+  int8_t      compressionLevel;
+  int32_t     daysPerFile;
   SValueNode* pDaysPerFile;
-  SValueNode* pFsyncPeriod;
-  SValueNode* pMaxRowsPerBlock;
-  SValueNode* pMinRowsPerBlock;
+  int32_t     fsyncPeriod;
+  int32_t     maxRowsPerBlock;
+  int32_t     minRowsPerBlock;
   SNodeList*  pKeep;
-  SValueNode* pPrecision;
-  SValueNode* pQuorum;
-  SValueNode* pReplica;
-  SValueNode* pTtl;
-  SValueNode* pWalLevel;
-  SValueNode* pNumOfVgroups;
-  SValueNode* pSingleStable;
-  SValueNode* pStreamMode;
-  SValueNode* pStrict;
+  int32_t     keep[3];
+  int32_t     pages;
+  int32_t     pagesize;
+  char        precisionStr[3];
+  int8_t      precision;
+  int8_t      replica;
+  int8_t      strict;
+  int8_t      walLevel;
+  int32_t     numOfVgroups;
+  int8_t      singleStable;
   SNodeList*  pRetentions;
 } SDatabaseOptions;
 
@@ -77,13 +78,12 @@ typedef struct SAlterDatabaseStmt {
 
 typedef struct STableOptions {
   ENodeType  type;
-  SNodeList* pKeep;
-  SValueNode* pTtl;
-  SValueNode* pComments;
+  char       comment[TSDB_STB_COMMENT_LEN];
+  int32_t    delay;
+  float      filesFactor;
+  SNodeList* pRollupFuncs;
+  int32_t    ttl;
   SNodeList* pSma;
-  SNodeList* pFuncs;
-  SValueNode* pFilesFactor;
-  SValueNode* pDelay;
 } STableOptions;
 
 typedef struct SColumnDefNode {
@@ -201,10 +201,7 @@ typedef struct SShowCreatStmt {
   char      tableName[TSDB_TABLE_NAME_LEN];
 } SShowCreatStmt;
 
-typedef enum EIndexType {
-  INDEX_TYPE_SMA = 1,
-  INDEX_TYPE_FULLTEXT
-} EIndexType;
+typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT } EIndexType;
 
 typedef struct SIndexOptions {
   ENodeType  type;

include/libs/nodes/plannodes.h

@@ -340,7 +340,7 @@ typedef struct SQueryPlan {
   int32_t      numOfSubplans;
   SNodeList*   pSubplans;  // Element is SNodeListNode. The execution level of subplan, starting from 0.
   SExplainInfo explainInfo;
-  SNodeList*   pPlaceholderValues;
+  SArray* pPlaceholderValues;
 } SQueryPlan;
 
 void nodesWalkPhysiPlan(SNode* pNode, FNodeWalker walker, void* pContext);

include/libs/parser/parser.h

@@ -73,6 +73,7 @@ typedef struct SQuery {
   SArray* pDbList;
   SArray* pTableList;
   bool showRewrite;
+  int32_t placeholderNum;
 } SQuery;
 
 int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery);

include/libs/planner/planner.h

@@ -34,7 +34,7 @@ typedef struct SPlanContext {
   bool showRewrite;
   int8_t triggerType;
   int64_t watermark;
-  bool isStmtQuery;
+  int32_t placeholderNum;
   void* pTransporter;
   struct SCatalog* pCatalog;
   char* pMsg;
@@ -50,7 +50,7 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo
 // @pSource one execution location of this group of datasource subplans 
 int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource);
 
-int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx);
+int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId);
 
 // Convert to subplan to string for the scheduler to send to the executor
 int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen);

include/libs/stream/tstream.h

@@ -35,19 +35,6 @@ enum {
   STREAM_CREATED_BY__SMA,
 };
 
-#if 0
-// pipe  -> fetch/pipe queue
-// merge -> merge      queue
-// write -> write      queue
-enum {
-  TASK_DISPATCH_MSG__SND_PIPE = 1,
-  TASK_DISPATCH_MSG__SND_MERGE,
-  TASK_DISPATCH_MSG__VND_PIPE,
-  TASK_DISPATCH_MSG__VND_MERGE,
-  TASK_DISPATCH_MSG__VND_WRITE,
-};
-#endif
-
 typedef struct {
   int32_t nodeId;  // 0 for snode
   SEpSet  epSet;
@@ -100,10 +87,6 @@ typedef struct {
   int8_t reserved;
 } STaskSinkFetch;
 
-typedef struct {
-  int8_t reserved;
-} STaskSinkShow;
-
 enum {
   TASK_SOURCE__SCAN = 1,
   TASK_SOURCE__PIPE,
@@ -128,7 +111,6 @@ enum {
   TASK_SINK__TABLE,
   TASK_SINK__SMA,
   TASK_SINK__FETCH,
-  TASK_SINK__SHOW,
 };
 
 typedef struct {
@@ -155,7 +137,6 @@ typedef struct {
     STaskSinkTb    tbSink;
     STaskSinkSma   smaSink;
     STaskSinkFetch fetchSink;
-    STaskSinkShow  showSink;
   };
 
   // dispatch

include/libs/stream/tstreamUpdate.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef _TSTREAMUPDATE_H_
+#define _TSTREAMUPDATE_H_
+
+#include "taosdef.h"
+#include "tarray.h"
+#include "tmsg.h"
+#include "tscalablebf.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SUpdateInfo {
+  SArray *pTsBuckets;
+  uint64_t numBuckets;
+  SArray *pTsSBFs;
+  uint64_t numSBFs;
+  int64_t interval;
+  int64_t watermark;
+  TSKEY minTS;
+} SUpdateInfo;
+
+SUpdateInfo *updateInfoInitP(SInterval* pInterval, int64_t watermark);
+SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark);
+bool isUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts);
+void updateInfoDestroy(SUpdateInfo *pInfo);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* ifndef _TSTREAMUPDATE_H_ */
\ No newline at end of file

include/os/os.h

@@ -59,6 +59,7 @@ extern "C" {
 #include <winsock.h>
 #endif
 
+#define __typeof(a) auto
 
 #endif
 

include/os/osMath.h

@@ -23,27 +23,21 @@ extern "C" {
 #define TPOW2(x) ((x) * (x))
 #define TABS(x) ((x) > 0 ? (x) : -(x))
 
-#ifdef WINDOWS
-
-  #define TSWAP(a, b, c) \
+#define TSWAP(a, b)             \
   do {                          \
-      c __tmp = (c)(a); \
-      (a) = (c)(b);     \
+    __typeof(a) __tmp = (a); \
+    (a) = (b);               \
     (b) = __tmp;                \
   } while (0)
+
+#ifdef WINDOWS
+
   #define TMAX(a, b) (((a) > (b)) ? (a) : (b))
   #define TMIN(a, b) (((a) < (b)) ? (a) : (b))
   #define TRANGE(aa, bb, cc) ((aa) = TMAX((aa), (bb)),(aa) = TMIN((aa), (cc)))
 
 #else
 
-  #define TSWAP(a, b, c)       \
-    do {                       \
-      __typeof(a) __tmp = (a); \
-      (a) = (b);               \
-      (b) = __tmp;             \
-    } while (0)
-
   #define TMAX(a, b)             \
     ({                           \
       __typeof(a) __a = (a);     \
@@ -51,7 +45,7 @@ extern "C" {
       (__a > __b) ? __a : __b;   \
     })
 
-  #define TMIN(a, b)             \
+#define TMIN(a, b)             \
   ({                           \
     __typeof(a) __a = (a);     \
     __typeof(b) __b = (b);     \

include/os/osSocket.h

@@ -19,53 +19,53 @@
 // If the error is in a third-party library, place this header file under the third-party library header file.
 // When you want to use this feature, you should find or add the same function in the following section.
 #ifndef ALLOW_FORBID_FUNC
-  #define socket SOCKET_FUNC_TAOS_FORBID
-  #define bind   BIND_FUNC_TAOS_FORBID
-  #define listen LISTEN_FUNC_TAOS_FORBID
-  #define accept ACCEPT_FUNC_TAOS_FORBID
-  #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID
-  #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID
-  #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID
-  #define inet_addr INET_ADDR_FUNC_TAOS_FORBID
-  #define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID
+#define socket       SOCKET_FUNC_TAOS_FORBID
+#define bind         BIND_FUNC_TAOS_FORBID
+#define listen       LISTEN_FUNC_TAOS_FORBID
+#define accept       ACCEPT_FUNC_TAOS_FORBID
+#define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID
+#define epoll_ctl    EPOLL_CTL_FUNC_TAOS_FORBID
+#define epoll_wait   EPOLL_WAIT_FUNC_TAOS_FORBID
+#define inet_addr    INET_ADDR_FUNC_TAOS_FORBID
+#define inet_ntoa    INET_NTOA_FUNC_TAOS_FORBID
 #endif
 
 #if defined(WINDOWS)
-  #if BYTE_ORDER == LITTLE_ENDIAN
-    #include <stdlib.h>
-    #define htobe16(x) _byteswap_ushort(x)
-    #define htole16(x) (x)
-    #define be16toh(x) _byteswap_ushort(x)
-    #define le16toh(x) (x)
-                      
-    #define htobe32(x) _byteswap_ulong(x)
-    #define htole32(x) (x)
-    #define be32toh(x) _byteswap_ulong(x)
-    #define le32toh(x) (x)
-                      
-    #define htobe64(x) _byteswap_uint64(x)
-    #define htole64(x) (x)
-    #define be64toh(x) _byteswap_uint64(x)
-    #define le64toh(x) (x)
-  #else
-    #error byte order not supported
-  #endif
-
-  #define __BYTE_ORDER    BYTE_ORDER
-  #define __BIG_ENDIAN    BIG_ENDIAN
-  #define __LITTLE_ENDIAN LITTLE_ENDIAN
-  #define __PDP_ENDIAN    PDP_ENDIAN
+#if BYTE_ORDER == LITTLE_ENDIAN
+#include <stdlib.h>
+#define htobe16(x) _byteswap_ushort(x)
+#define htole16(x) (x)
+#define be16toh(x) _byteswap_ushort(x)
+#define le16toh(x) (x)
+
+#define htobe32(x) _byteswap_ulong(x)
+#define htole32(x) (x)
+#define be32toh(x) _byteswap_ulong(x)
+#define le32toh(x) (x)
+
+#define htobe64(x) _byteswap_uint64(x)
+#define htole64(x) (x)
+#define be64toh(x) _byteswap_uint64(x)
+#define le64toh(x) (x)
+#else
+#error byte order not supported
+#endif
+
+#define __BYTE_ORDER    BYTE_ORDER
+#define __BIG_ENDIAN    BIG_ENDIAN
+#define __LITTLE_ENDIAN LITTLE_ENDIAN
+#define __PDP_ENDIAN    PDP_ENDIAN
 
 #else
-  #include <netinet/in.h>
-  #include <sys/socket.h>
-
-  #if defined(_TD_DARWIN_64)
-    #include <osEok.h>
-  #else
-    #include <netinet/in.h>
-    #include <sys/epoll.h>
-  #endif
+#include <netinet/in.h>
+#include <sys/socket.h>
+
+#if defined(_TD_DARWIN_64)
+#include <osEok.h>
+#else
+#include <netinet/in.h>
+#include <sys/epoll.h>
+#endif
 #endif
 
 #ifdef __cplusplus
@@ -73,24 +73,24 @@ extern "C" {
 #endif
 
 #if defined(WINDOWS)
-  typedef int socklen_t;
-  #define TAOS_EPOLL_WAIT_TIME 100
-  typedef SOCKET eventfd_t;
-  #define eventfd(a, b) -1
-  #define EpollClose(pollFd) epoll_close(pollFd)
-  #ifndef EPOLLWAKEUP
-    #define EPOLLWAKEUP (1u << 29)
-  #endif
+typedef int socklen_t;
+#define TAOS_EPOLL_WAIT_TIME 100
+typedef SOCKET eventfd_t;
+#define eventfd(a, b)      -1
+#define EpollClose(pollFd) epoll_close(pollFd)
+#ifndef EPOLLWAKEUP
+#define EPOLLWAKEUP (1u << 29)
+#endif
 #elif defined(_TD_DARWIN_64)
-  #define TAOS_EPOLL_WAIT_TIME 500
-  typedef int32_t SOCKET;
-  typedef SOCKET EpollFd;
-  #define EpollClose(pollFd) epoll_close(pollFd)
+#define TAOS_EPOLL_WAIT_TIME 500
+typedef int32_t SOCKET;
+typedef SOCKET  EpollFd;
+#define EpollClose(pollFd)   epoll_close(pollFd)
 #else
-  #define TAOS_EPOLL_WAIT_TIME 500
-  typedef int32_t SOCKET;
-  typedef SOCKET EpollFd;
-  #define EpollClose(pollFd) taosCloseSocket(pollFd)
+#define TAOS_EPOLL_WAIT_TIME 500
+typedef int32_t SOCKET;
+typedef SOCKET  EpollFd;
+#define EpollClose(pollFd)   taosCloseSocket(pollFd)
 #endif
 
 #if defined(_TD_DARWIN_64)
@@ -128,16 +128,17 @@ typedef struct TdSocket {
 #endif
   int      refId;
   SocketFd fd;
-} *TdSocketPtr, TdSocket;
+} * TdSocketPtr, TdSocket;
 
 typedef struct TdSocketServer *TdSocketServerPtr;
-typedef struct TdSocket *TdSocketPtr;
-typedef struct TdEpoll *TdEpollPtr;
+typedef struct TdSocket *      TdSocketPtr;
+typedef struct TdEpoll *       TdEpollPtr;
 
-int32_t taosSendto(TdSocketPtr pSocket, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen);
+int32_t taosSendto(TdSocketPtr pSocket, void *msg, int len, unsigned int flags, const struct sockaddr *to, int tolen);
 int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len);
 int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len);
-int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, int *addrLen);
+int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr,
+                           int *addrLen);
 int32_t taosCloseSocketNoCheck1(SocketFd fd);
 int32_t taosCloseSocket(TdSocketPtr *ppSocket);
 int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer);
@@ -156,13 +157,15 @@ int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes);
 int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len);
 void    taosWinSocketInit();
 
+int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec);
+
 TdSocketPtr       taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
 TdSocketPtr       taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
 TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
 int32_t           taosKeepTcpAlive(TdSocketPtr pSocket);
 TdSocketPtr       taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen);
 
-int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, int *addrLen);
+int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, int *addrLen);
 
 void        taosBlockSIGPIPE();
 uint32_t    taosGetIpv4FromFqdn(const char *);

include/util/taoserror.h

@@ -62,6 +62,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_APP_NOT_READY                 TAOS_DEF_ERROR_CODE(0, 0x0014)
 #define TSDB_CODE_RPC_FQDN_ERROR                TAOS_DEF_ERROR_CODE(0, 0x0015)
 #define TSDB_CODE_RPC_INVALID_VERSION           TAOS_DEF_ERROR_CODE(0, 0x0016)
+#define TSDB_CODE_RPC_PORT_EADDRINUSE           TAOS_DEF_ERROR_CODE(0, 0x0017)
 
 //common & util
 #define TSDB_CODE_OUT_OF_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0100)
@@ -243,9 +244,10 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_TOO_MANY_COLUMNS          TAOS_DEF_ERROR_CODE(0, 0x03AC)
 #define TSDB_CODE_MND_COLUMN_ALREADY_EXIST      TAOS_DEF_ERROR_CODE(0, 0x03AD)
 #define TSDB_CODE_MND_COLUMN_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03AE)
+#define TSDB_CODE_MND_SINGLE_STB_MODE_DB        TAOS_DEF_ERROR_CODE(0, 0x03B0)
 
 // mnode-infoSchema
-#define TSDB_CODE_MND_INVALID_SYS_TABLENAME         TAOS_DEF_ERROR_CODE(0, 0x03B0)
+#define TSDB_CODE_MND_INVALID_SYS_TABLENAME         TAOS_DEF_ERROR_CODE(0, 0x03BA)
 
 // mnode-func
 #define TSDB_CODE_MND_FUNC_ALREADY_EXIST        TAOS_DEF_ERROR_CODE(0, 0x03C0)
@@ -268,14 +270,13 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_TOPIC_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x03E1)
 #define TSDB_CODE_MND_TOO_MANY_TOPICS           TAOS_DEF_ERROR_CODE(0, 0x03E2)
 #define TSDB_CODE_MND_INVALID_TOPIC             TAOS_DEF_ERROR_CODE(0, 0x03E3)
-#define TSDB_CODE_MND_INVALID_TOPIC_OPTION      TAOS_DEF_ERROR_CODE(0, 0x03E4)
-#define TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED    TAOS_DEF_ERROR_CODE(0, 0x03E5)
-#define TSDB_CODE_MND_NAME_CONFLICT_WITH_STB    TAOS_DEF_ERROR_CODE(0, 0x03E6)
-#define TSDB_CODE_MND_CONSUMER_NOT_EXIST        TAOS_DEF_ERROR_CODE(0, 0x03E7)
-#define TSDB_CODE_MND_UNSUPPORTED_TOPIC         TAOS_DEF_ERROR_CODE(0, 0x03E8)
-#define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E9)
-#define TSDB_CODE_MND_OFFSET_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03EA)
-#define TSDB_CODE_MND_CONSUMER_NOT_READY        TAOS_DEF_ERROR_CODE(0, 0x03EB)
+#define TSDB_CODE_MND_INVALID_TOPIC_QUERY       TAOS_DEF_ERROR_CODE(0, 0x03E4)
+#define TSDB_CODE_MND_INVALID_TOPIC_OPTION      TAOS_DEF_ERROR_CODE(0, 0x03E5)
+#define TSDB_CODE_MND_CONSUMER_NOT_EXIST        TAOS_DEF_ERROR_CODE(0, 0x03E6)
+#define TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED    TAOS_DEF_ERROR_CODE(0, 0x03E7)
+#define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E8)
+#define TSDB_CODE_MND_OFFSET_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03E9)
+#define TSDB_CODE_MND_CONSUMER_NOT_READY        TAOS_DEF_ERROR_CODE(0, 0x03EA)
 
 // mnode-stream
 #define TSDB_CODE_MND_STREAM_ALREADY_EXIST      TAOS_DEF_ERROR_CODE(0, 0x03F0)
@@ -586,7 +587,6 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INVALID_RANGE_OPTION      TAOS_DEF_ERROR_CODE(0, 0x2619)
 #define TSDB_CODE_PAR_INVALID_STR_OPTION        TAOS_DEF_ERROR_CODE(0, 0x261A)
 #define TSDB_CODE_PAR_INVALID_ENUM_OPTION       TAOS_DEF_ERROR_CODE(0, 0x261B)
-#define TSDB_CODE_PAR_INVALID_TTL_OPTION        TAOS_DEF_ERROR_CODE(0, 0x261C)
 #define TSDB_CODE_PAR_INVALID_KEEP_NUM          TAOS_DEF_ERROR_CODE(0, 0x261D)
 #define TSDB_CODE_PAR_INVALID_KEEP_ORDER        TAOS_DEF_ERROR_CODE(0, 0x261E)
 #define TSDB_CODE_PAR_INVALID_KEEP_VALUE        TAOS_DEF_ERROR_CODE(0, 0x261F)

include/util/tarray.h

@@ -205,7 +205,6 @@ SArray* taosArrayDup(const SArray* pSrc);
  */
 SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy);
 
-
 /**
  * clear the array (remove all element)
  * @param pArray
@@ -219,6 +218,13 @@ void taosArrayClear(SArray* pArray);
  */
 void taosArrayClearEx(SArray* pArray, void (*fp)(void*));
 
+/**
+ * clear the array (remove all element)
+ * @param pArray
+ * @param fp
+ */
+void taosArrayClearP(SArray* pArray, FDelete fp);
+
 void* taosArrayDestroy(SArray* pArray);
 void  taosArrayDestroyP(SArray* pArray, FDelete fp);
 void  taosArrayDestroyEx(SArray* pArray, FDelete fp);
@@ -272,6 +278,8 @@ void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* par
 int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode);
 void*   taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz);
 
+char* taosShowStrArray(const SArray* pArray);
+
 #ifdef __cplusplus
 }
 #endif

include/util/tbloomfilter.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_UTIL_BLOOMFILTER_H_
+#define _TD_UTIL_BLOOMFILTER_H_
+
+#include "os.h"
+#include "thash.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SBloomFilter {
+  uint32_t hashFunctions;
+  uint64_t expectedEntries;
+  uint64_t numUnits;
+  uint64_t numBits;
+  uint64_t size;
+  _hash_fn_t hashFn1;
+  _hash_fn_t hashFn2;
+  void *buffer;
+  double errorRate;
+} SBloomFilter;
+
+SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate);
+int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len);
+int32_t tBloomFilterNoContain(const SBloomFilter *pBF, const void *keyBuf,
+                              uint32_t len);
+void tBloomFilterDestroy(SBloomFilter *pBF);
+void tBloomFilterDump(const SBloomFilter *pBF);
+bool tBloomFilterIsFull(const SBloomFilter *pBF);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_BLOOMFILTER_H_*/
\ No newline at end of file

include/util/tdef.h

@@ -97,41 +97,6 @@ extern const int32_t TYPE_BYTES[15];
 #define TSDB_TIME_PRECISION_MICRO_DIGITS 16
 #define TSDB_TIME_PRECISION_NANO_DIGITS  19
 
-#define TSDB_INFORMATION_SCHEMA_DB            "information_schema"
-#define TSDB_PERFORMANCE_SCHEMA_DB            "performance_schema"
-#define TSDB_INS_TABLE_DNODES                 "dnodes"
-#define TSDB_INS_TABLE_MNODES                 "mnodes"
-#define TSDB_INS_TABLE_MODULES                "modules"
-#define TSDB_INS_TABLE_QNODES                 "qnodes"
-#define TSDB_INS_TABLE_BNODES                 "bnodes"
-#define TSDB_INS_TABLE_SNODES                 "snodes"
-#define TSDB_INS_TABLE_CLUSTER                "cluster"
-#define TSDB_INS_TABLE_USER_DATABASES         "user_databases"
-#define TSDB_INS_TABLE_USER_FUNCTIONS         "user_functions"
-#define TSDB_INS_TABLE_USER_INDEXES           "user_indexes"
-#define TSDB_INS_TABLE_USER_STABLES           "user_stables"
-#define TSDB_INS_TABLE_USER_STREAMS           "user_streams"
-#define TSDB_INS_TABLE_USER_TABLES            "user_tables"
-#define TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED "user_table_distributed"
-#define TSDB_INS_TABLE_USER_USERS             "user_users"
-#define TSDB_INS_TABLE_LICENCES               "grants"
-#define TSDB_INS_TABLE_VGROUPS                "vgroups"
-#define TSDB_INS_TABLE_CONSUMERS              "consumers"
-#define TSDB_INS_TABLE_SUBSCRIBES             "subscribes"
-#define TSDB_INS_TABLE_TRANS                  "trans"
-#define TSDB_INS_TABLE_SMAS                   "smas"
-#define TSDB_INS_TABLE_CONFIGS                "configs"
-#define TSDB_INS_TABLE_CONNS                  "connections"
-#define TSDB_INS_TABLE_QUERIES                "queries"
-#define TSDB_INS_TABLE_VNODES                 "vnodes"
-
-#define TSDB_PERFORMANCE_SCHEMA_DB     "performance_schema"
-#define TSDB_PERFS_TABLE_CONNECTIONS   "connections"
-#define TSDB_PERFS_TABLE_QUERIES       "queries"
-#define TSDB_PERFS_TABLE_TOPICS        "topics"
-#define TSDB_PERFS_TABLE_CONSUMERS     "consumers"
-#define TSDB_PERFS_TABLE_SUBSCRIPTIONS "subscriptions"
-
 #define TSDB_INDEX_TYPE_SMA      "SMA"
 #define TSDB_INDEX_TYPE_FULLTEXT "FULLTEXT"
 
@@ -203,16 +168,6 @@ typedef enum ELogicConditionType {
   LOGIC_COND_TYPE_NOT,
 } ELogicConditionType;
 
-#define FUNCTION_CEIL  4500
-#define FUNCTION_FLOOR 4501
-#define FUNCTION_ABS   4502
-#define FUNCTION_ROUND 4503
-
-#define FUNCTION_LENGTH 4800
-#define FUNCTION_CONCAT 4801
-#define FUNCTION_LTRIM  4802
-#define FUNCTION_RTRIM  4803
-
 #define TSDB_NAME_DELIMITER_LEN 1
 
 #define TSDB_UNI_LEN  24
@@ -325,15 +280,21 @@ typedef enum ELogicConditionType {
 #define TSDB_MIN_VNODES_PER_DB          1
 #define TSDB_MAX_VNODES_PER_DB          4096
 #define TSDB_DEFAULT_VN_PER_DB          2
-#define TSDB_MIN_CACHE_BLOCK_SIZE     1
-#define TSDB_MAX_CACHE_BLOCK_SIZE     128  // 128MB for each vnode
-#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16
-#define TSDB_MIN_TOTAL_BLOCKS         3
-#define TSDB_MAX_TOTAL_BLOCKS         10000
-#define TSDB_DEFAULT_TOTAL_BLOCKS     6
+#define TSDB_MIN_BUFFER_PER_VNODE       3      // unit MB
+#define TSDB_MAX_BUFFER_PER_VNODE       16384  // unit MB
+#define TSDB_DEFAULT_BUFFER_PER_VNODE   96
+#define TSDB_MIN_PAGES_PER_VNODE        64
+#define TSDB_MAX_PAGES_PER_VNODE        16384
+#define TSDB_DEFAULT_PAGES_PER_VNODE    256
+#define TSDB_MIN_PAGESIZE_PER_VNODE     1  // unit KB
+#define TSDB_MAX_PAGESIZE_PER_VNODE     16384
+#define TSDB_DEFAULT_PAGESIZE_PER_VNODE 4
 #define TSDB_MIN_DAYS_PER_FILE          60  // unit minute
 #define TSDB_MAX_DAYS_PER_FILE          (3650 * 1440)
 #define TSDB_DEFAULT_DAYS_PER_FILE      (10 * 1440)
+#define TSDB_MIN_DURATION_PER_FILE      60  // unit minute
+#define TSDB_MAX_DURATION_PER_FILE      (3650 * 1440)
+#define TSDB_DEFAULT_DURATION_PER_FILE  (10 * 1440)
 #define TSDB_MIN_KEEP                   (1 * 1440)       // data in db to be reserved. unit minute
 #define TSDB_MAX_KEEP                   (365000 * 1440)  // data in db to be reserved.
 #define TSDB_DEFAULT_KEEP               (3650 * 1440)    // ten years
@@ -343,14 +304,9 @@ typedef enum ELogicConditionType {
 #define TSDB_MIN_MAXROWS_FBLOCK         200
 #define TSDB_MAX_MAXROWS_FBLOCK         10000
 #define TSDB_DEFAULT_MAXROWS_FBLOCK     4096
-#define TSDB_MIN_COMMIT_TIME          30
-#define TSDB_MAX_COMMIT_TIME          40960
-#define TSDB_DEFAULT_COMMIT_TIME      3600
 #define TSDB_MIN_FSYNC_PERIOD           0
 #define TSDB_MAX_FSYNC_PERIOD           180000  // millisecond
 #define TSDB_DEFAULT_FSYNC_PERIOD       3000    // three second
-#define TSDB_MIN_DB_TTL               1
-#define TSDB_DEFAULT_DB_TTL           1
 #define TSDB_MIN_WAL_LEVEL              1
 #define TSDB_MAX_WAL_LEVEL              2
 #define TSDB_DEFAULT_WAL_LEVEL          1
@@ -366,9 +322,6 @@ typedef enum ELogicConditionType {
 #define TSDB_DB_STRICT_OFF              0
 #define TSDB_DB_STRICT_ON               1
 #define TSDB_DEFAULT_DB_STRICT          0
-#define TSDB_MIN_DB_UPDATE            0
-#define TSDB_MAX_DB_UPDATE            2
-#define TSDB_DEFAULT_DB_UPDATE        0
 #define TSDB_MIN_DB_CACHE_LAST_ROW      0
 #define TSDB_MAX_DB_CACHE_LAST_ROW      3
 #define TSDB_DEFAULT_CACHE_LAST_ROW     0
@@ -379,12 +332,15 @@ typedef enum ELogicConditionType {
 #define TSDB_DB_SINGLE_STABLE_OFF       1
 #define TSDB_DEFAULT_DB_SINGLE_STABLE   0
 
-#define TSDB_MIN_DB_FILE_FACTOR     0
-#define TSDB_MAX_DB_FILE_FACTOR     1
-#define TSDB_DEFAULT_DB_FILE_FACTOR 0.1
-#define TSDB_MIN_DB_DELAY           1
-#define TSDB_MAX_DB_DELAY           10
-#define TSDB_DEFAULT_DB_DELAY       2
+#define TSDB_MIN_ROLLUP_FILE_FACTOR     0
+#define TSDB_MAX_ROLLUP_FILE_FACTOR     1
+#define TSDB_DEFAULT_ROLLUP_FILE_FACTOR 0.1
+#define TSDB_MIN_ROLLUP_DELAY           1
+#define TSDB_MAX_ROLLUP_DELAY           10
+#define TSDB_DEFAULT_ROLLUP_DELAY       2
+#define TSDB_MIN_TABLE_TTL              0
+#define TSDB_DEFAULT_TABLE_TTL          0
+
 #define TSDB_MIN_EXPLAIN_RATIO     0
 #define TSDB_MAX_EXPLAIN_RATIO     1
 #define TSDB_DEFAULT_EXPLAIN_RATIO 0.001
@@ -414,21 +370,9 @@ typedef enum ELogicConditionType {
  * 1. ordinary sub query for select * from super_table
  * 2. all sqlobj generated by createSubqueryObj with this flag
  */
-#define TSDB_QUERY_TYPE_SUBQUERY        0x02u
-#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u  // two-stage subquery for super table
-
-#define TSDB_QUERY_TYPE_TABLE_QUERY      0x08u  // query ordinary table; below only apply to client side
-#define TSDB_QUERY_TYPE_STABLE_QUERY     0x10u  // query on super table
-#define TSDB_QUERY_TYPE_JOIN_QUERY       0x20u  // join query
-#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u  // select *,columns... query
-#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE   0x80u  // join sub query at the second stage
-
-#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u
 #define TSDB_QUERY_TYPE_INSERT           0x100u  // insert type
-#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u
 #define TSDB_QUERY_TYPE_FILE_INSERT      0x400u   // insert data from file
 #define TSDB_QUERY_TYPE_STMT_INSERT      0x800u   // stmt insert type
-#define TSDB_QUERY_TYPE_NEST_SUBQUERY    0x1000u  // nested sub query
 
 #define TSDB_QUERY_HAS_TYPE(x, _type)   (((x) & (_type)) != 0)
 #define TSDB_QUERY_SET_TYPE(x, _type)   ((x) |= (_type))

include/util/tencode.h

@@ -79,31 +79,52 @@ typedef struct {
 #define TD_CODER_CURRENT(CODER)                        ((CODER)->data + (CODER)->pos)
 #define TD_CODER_MOVE_POS(CODER, MOVE)                 ((CODER)->pos += (MOVE))
 #define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE))
-#define TCODER_MALLOC(PCODER, SIZE)                                         \
-  ({                                                                        \
-    void*      ptr = NULL;                                                  \
-    SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(*pMem) + (SIZE)); \
-    if (pMem) {                                                             \
-      pMem->next = (PCODER)->mList;                                         \
-      (PCODER)->mList = pMem;                                               \
-      ptr = (void*)&pMem[1];                                                \
-    }                                                                       \
-    ptr;                                                                    \
-  })
+// #define TCODER_MALLOC(PCODER, SIZE)                                         \
+//   ({                                                                        \
+//     void*      ptr = NULL;                                                  \
+//     SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(*pMem) + (SIZE)); \
+//     if (pMem) {                                                             \
+//       pMem->next = (PCODER)->mList;                                         \
+//       (PCODER)->mList = pMem;                                               \
+//       ptr = (void*)&pMem[1];                                                \
+//     }                                                                       \
+//     ptr;                                                                    \
+//   })
+static FORCE_INLINE void* tCoderMalloc(SCoder* pCoder, int32_t size) {
+  void*      ptr = NULL;
+  SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(SCoderMem*) + size);
+  if (pMem) {
+    pMem->next = pCoder->mList;
+    pCoder->mList = pMem;
+    ptr = (void*)&pMem[1];
+  }
+  return ptr;
+}
 
-#define tEncodeSize(E, S, SIZE)                                \
-  ({                                                           \
+#define tEncodeSize(E, S, SIZE, RET)                           \
+  do{                                                          \
     SCoder coder = {0};                                        \
-    int    ret = 0;                                            \
     tCoderInit(&coder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); \
     if ((E)(&coder, S) == 0) {                                 \
       SIZE = coder.pos;                                        \
     } else {                                                   \
-      ret = -1;                                                \
+      RET = -1;                                                \
     }                                                          \
     tCoderClear(&coder);                                       \
-    ret;                                                       \
-  })
+  }while(0)
+// #define tEncodeSize(E, S, SIZE)                                \
+//   ({                                                           \
+//     SCoder coder = {0};                                        \
+//     int    ret = 0;                                            \
+//     tCoderInit(&coder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); \
+//     if ((E)(&coder, S) == 0) {                                 \
+//       SIZE = coder.pos;                                        \
+//     } else {                                                   \
+//       ret = -1;                                                \
+//     }                                                          \
+//     tCoderClear(&coder);                                       \
+//     ret;                                                       \
+//   })
 
 void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type);
 void tCoderClear(SCoder* pCoder);

include/util/tscalablebf.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_UTIL_SCALABLEBF_H_
+#define _TD_UTIL_SCALABLEBF_H_
+
+#include "tbloomfilter.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SScalableBf {
+  SArray *bfArray; // array of bloom filters
+  uint32_t growth;
+  uint64_t numBits;
+} SScalableBf;
+
+SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate);
+int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len);
+int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf,
+                             uint32_t len);
+void tScalableBfDestroy(SScalableBf *pSBf);
+void tScalableBfDump(const SScalableBf *pSBf);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_SCALABLEBF_H_*/
\ No newline at end of file

source/client/inc/clientInt.h

@@ -254,8 +254,6 @@ extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t
 int           genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code);
 SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj);
 
-int taos_init();
-
 void*    createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo);
 void     destroyTscObj(void* pObj);
 STscObj* acquireTscObj(int64_t rid);
@@ -313,7 +311,6 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code
 int32_t      getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList);
 int32_t      scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList);
 
-
 #ifdef __cplusplus
 }
 #endif

source/client/inc/clientStmt.h

@@ -46,6 +46,12 @@ typedef struct SStmtTableCache {
   void*             boundTags;
 } SStmtTableCache;
 
+typedef struct SQueryFields {
+  TAOS_FIELD*    fields;
+  TAOS_FIELD*    userFields;
+  uint32_t       numOfCols;
+} SQueryFields;
+
 typedef struct SStmtBindInfo {
   bool         needParse;
   uint64_t     tbUid;
@@ -76,6 +82,7 @@ typedef struct SStmtSQLInfo {
   int32_t       sqlLen;
   SArray*       nodeList;
   SQueryPlan*   pQueryPlan;
+  SQueryFields  fields;
 } SStmtSQLInfo;
 
 typedef struct STscStmt {

source/client/src/clientImpl.c

@@ -103,6 +103,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass,
 
   if (port) {
     epSet.epSet.eps[0].port = port;
+    epSet.epSet.eps[1].port = port;
   }
 
   char* key = getClusterKey(user, secretEncrypt, ip, port);
@@ -187,8 +188,8 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC
       setResPrecision(&pRequest->body.resInfo, (*pQuery)->precision);
     }
 
-    TSWAP(pRequest->dbList, (*pQuery)->pDbList, SArray*);
-    TSWAP(pRequest->tableList, (*pQuery)->pTableList, SArray*);
+    TSWAP(pRequest->dbList, (*pQuery)->pDbList);
+    TSWAP(pRequest->tableList, (*pQuery)->pTableList);
   }
 
   return code;
@@ -233,7 +234,8 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra
                       .showRewrite = pQuery->showRewrite,
                       .pTransporter = pRequest->pTscObj->pAppInfo->pTransporter,
                       .pMsg = pRequest->msgBuf,
-                      .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE};
+                      .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE,
+                      .placeholderNum = pQuery->placeholderNum};
   int32_t      code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &cxt.pCatalog);
   if (TSDB_CODE_SUCCESS == code) {
     code = qCreateQueryPlan(&cxt, pPlan, pNodeList);
@@ -245,7 +247,6 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t
   ASSERT(pSchema != NULL && numOfCols > 0);
 
   pResInfo->numOfCols = numOfCols;
-  // TODO handle memory leak
   if (pResInfo->fields != NULL) {
     taosMemoryFree(pResInfo->fields);
   }
@@ -358,8 +359,15 @@ SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
   SQuery*      pQuery = NULL;
 
   int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest);
-  if (TSDB_CODE_SUCCESS == code) {
+  if (code != TSDB_CODE_SUCCESS) {
+    terrno = code;
+    return NULL;
+  }
+
   code = parseSql(pRequest, false, &pQuery, NULL);
+  if (code != TSDB_CODE_SUCCESS) {
+    pRequest->code = code;
+    return pRequest;
   }
 
   return launchQueryImpl(pRequest, pQuery, code, false);
@@ -410,7 +418,7 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
 
   while (retryNum++ < REQUEST_MAX_TRY_TIMES) {
     pRequest = launchQuery(pTscObj, sql, sqlLen);
-    if (TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) {
+    if (pRequest == NULL || TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) {
       break;
     }
 

source/client/src/clientMain.c

@@ -110,16 +110,23 @@ int taos_errno(TAOS_RES *tres) {
     return terrno;
   }
 
+  if (TD_RES_TMQ(tres)) {
+    return 0;
+  }
+
   return ((SRequestObj *)tres)->code;
 }
 
 const char *taos_errstr(TAOS_RES *res) {
-  SRequestObj *pRequest = (SRequestObj *)res;
-
-  if (pRequest == NULL) {
+  if (res == NULL) {
     return (const char *)tstrerror(terrno);
   }
 
+  if (TD_RES_TMQ(res)) {
+    return "success";
+  }
+
+  SRequestObj *pRequest = (SRequestObj *)res;
   if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) {
     return pRequest->msgBuf;
   } else {
@@ -632,9 +639,7 @@ int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) {
   return stmtSetTbName(stmt, name);
 }
 
-int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name) {
-  return taos_stmt_set_tbname(stmt, name);
-}
+int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name) { return taos_stmt_set_tbname(stmt, name); }
 
 int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) {
   if (stmt == NULL || bind == NULL) {
@@ -750,9 +755,7 @@ TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt) {
   return stmtUseResult(stmt);
 }
 
-char *taos_stmt_errstr(TAOS_STMT *stmt) {
-  return (char *)stmtErrstr(stmt);
-}
+char *taos_stmt_errstr(TAOS_STMT *stmt) { return (char *)stmtErrstr(stmt); }
 
 int taos_stmt_affected_rows(TAOS_STMT *stmt) {
   if (stmt == NULL) {

source/client/src/clientStmt.c

@@ -73,6 +73,22 @@ int32_t stmtGetTbName(TAOS_STMT *stmt, char **tbName) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t stmtBackupQueryFields(STscStmt* pStmt) {
+  SQueryFields *pFields = &pStmt->sql.fields;
+  int32_t size = pFields->numOfCols * sizeof(TAOS_FIELD);
+  
+  pFields->numOfCols = pStmt->exec.pRequest->body.resInfo.numOfCols;
+  pFields->fields = taosMemoryMalloc(size);
+  pFields->userFields = taosMemoryMalloc(size);
+  if (NULL == pFields->fields || NULL == pFields->userFields) {
+    STMT_ERR_RET(TSDB_CODE_TSC_OUT_OF_MEMORY);
+  }
+  memcpy(pFields->fields, pStmt->exec.pRequest->body.resInfo.fields, size);
+  memcpy(pFields->userFields, pStmt->exec.pRequest->body.resInfo.userFields, size);
+
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t stmtSetBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
@@ -258,37 +274,42 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
   STableMeta *pTableMeta = NULL;
   SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp);
   STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta));
+  uint64_t uid = pTableMeta->uid;
+  uint64_t suid = pTableMeta->suid;
+  int8_t tableType = pTableMeta->tableType;
+  taosMemoryFree(pTableMeta);
   
-  if (pTableMeta->uid == pStmt->bInfo.tbUid) {
+  if (uid == pStmt->bInfo.tbUid) {
     pStmt->bInfo.needParse = false;
 
     return TSDB_CODE_SUCCESS;
   }
 
-  if (taosHashGet(pStmt->exec.pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid))) {
-    SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pTableMeta->uid, sizeof(pTableMeta->uid));
+  if (taosHashGet(pStmt->exec.pBlockHash, &uid, sizeof(uid))) {
+    SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid));
     if (NULL == pCache) {
-      tscError("table uid %" PRIx64 "found in exec blockHash, but not in sql blockHash", pTableMeta->uid);
+      tscError("table uid %" PRIx64 "found in exec blockHash, but not in sql blockHash", uid);
+      
       STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR);
     }
     
     pStmt->bInfo.needParse = false;
     
-    pStmt->bInfo.tbUid = pTableMeta->uid;
-    pStmt->bInfo.tbSuid = pTableMeta->suid;
-    pStmt->bInfo.tbType = pTableMeta->tableType;
+    pStmt->bInfo.tbUid = uid;
+    pStmt->bInfo.tbSuid = suid;
+    pStmt->bInfo.tbType = tableType;
     pStmt->bInfo.boundTags = pCache->boundTags;
 
     return TSDB_CODE_SUCCESS;
   }
 
-  SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pTableMeta->uid, sizeof(pTableMeta->uid));
+  SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid));
   if (pCache) {
     pStmt->bInfo.needParse = false;
 
-    pStmt->bInfo.tbUid = pTableMeta->uid;
-    pStmt->bInfo.tbSuid = pTableMeta->suid;
-    pStmt->bInfo.tbType = pTableMeta->tableType;
+    pStmt->bInfo.tbUid = uid;
+    pStmt->bInfo.tbSuid = suid;
+    pStmt->bInfo.tbType = tableType;
     pStmt->bInfo.boundTags = pCache->boundTags;
 
     STableDataBlocks* pNewBlock = NULL;
@@ -475,9 +496,10 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) {
       STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList));
       pStmt->sql.pQueryPlan = pStmt->exec.pRequest->body.pDag;
       pStmt->exec.pRequest->body.pDag = NULL;
+      STMT_ERR_RET(stmtBackupQueryFields(pStmt));
     }
     
-    STMT_RET(qStmtBindParam(pStmt->sql.pQueryPlan, bind, colIdx));
+    STMT_RET(qStmtBindParam(pStmt->sql.pQueryPlan, bind, colIdx, pStmt->exec.pRequest->requestId));
   }
   
   STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid));
@@ -549,6 +571,8 @@ int stmtClose(TAOS_STMT *stmt) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
   STMT_RET(stmtCleanSQLInfo(pStmt));
+
+  taosMemoryFree(stmt);
 }
 
 const char *stmtErrstr(TAOS_STMT *stmt) {
@@ -601,7 +625,7 @@ int stmtGetParamNum(TAOS_STMT *stmt, int *nums) {
       pStmt->exec.pRequest->body.pDag = NULL;
     }
 
-    *nums = (pStmt->sql.pQueryPlan->pPlaceholderValues) ? pStmt->sql.pQueryPlan->pPlaceholderValues->length : 0;
+    *nums = taosArrayGetSize(pStmt->sql.pQueryPlan->pPlaceholderValues);
   } else {
     STMT_ERR_RET(stmtFetchColFields(stmt, nums, NULL));
   }

source/client/src/taos.def

 taos_cleanup
 taos_options
 taos_set_config
+taos_init
 taos_connect
 taos_connect_l
 taos_connect_auth

source/client/src/tmq.c

@@ -666,9 +666,8 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
   code = param.rspErr;
   if (code != 0) goto FAIL;
 
-  // TODO: add max retry cnt
   while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) {
-    tscDebug("not ready, retry");
+    tscDebug("consumer not ready, retry");
     taosMsleep(500);
   }
 
@@ -683,7 +682,7 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
   code = 0;
 FAIL:
   if (req.topicNames != NULL) taosArrayDestroyP(req.topicNames, taosMemoryFree);
-  if (code != 0) {
+  if (code != 0 && buf) {
     taosMemoryFree(buf);
   }
   return code;
@@ -694,6 +693,7 @@ void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) {
   conf->commitCb = cb;
 }
 
+#if 0
 TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbName, const char* sql) {
   STscObj*     pTscObj = (STscObj*)taos;
   SRequestObj* pRequest = NULL;
@@ -740,7 +740,7 @@ TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbNa
       .sql = (char*)sql,
   };
   tNameExtractFullName(&name, req.name);
-  strcpy(req.outputSTbName, tbName);
+  strcpy(req.targetStbFullName, tbName);
 
   int   tlen = tSerializeSCMCreateStreamReq(NULL, 0, &req);
   void* buf = taosMemoryMalloc(tlen);
@@ -778,6 +778,7 @@ _return:
 
   return pRequest;
 }
+#endif
 
 #if 0
 int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) {
@@ -1265,6 +1266,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t wait_time) {
     return (TAOS_RES*)rspObj;
   }
 
+  // in no topic status also need process delayed task
   if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__INIT) {
     return NULL;
   }
@@ -1285,6 +1287,9 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t wait_time) {
         return NULL;
       }
       tsem_timewait(&tmq->rspSem, leftTime * 1000);
+    } else {
+      // use tsem_timewait instead of tsem_wait to avoid unexpected stuck
+      tsem_timewait(&tmq->rspSem, 500 * 1000);
     }
   }
 }

source/client/test/clientTests.cpp

@@ -659,10 +659,15 @@ TEST(testCase, agg_query_tables) {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   ASSERT_NE(pConn, nullptr);
 
-  TAOS_RES* pRes = taos_query(pConn, "use abc1");
+  TAOS_RES* pRes = taos_query(pConn, "use db");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to use db, reason:%s\n", taos_errstr(pRes));
+    taos_free_result(pRes);
+    ASSERT_TRUE(false);
+  }
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "select now() from m1");
+  pRes = taos_query(pConn, "select tbname from st1");
   if (taos_errno(pRes) != 0) {
     printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
     taos_free_result(pRes);

source/common/src/tdatablock.c

@@ -225,13 +225,17 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co
     // Handle the bitmap
     char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2));
     if (p == NULL) {
-      // TODO
+      return TSDB_CODE_OUT_OF_MEMORY;
     }
 
     pColumnInfoData->varmeta.offset = (int32_t*)p;
     for (int32_t i = 0; i < numOfRow2; ++i) {
+      if (pSource->varmeta.offset[i] == -1) {
+        pColumnInfoData->varmeta.offset[i + numOfRow1] = -1;
+      } else {
         pColumnInfoData->varmeta.offset[i + numOfRow1] = pSource->varmeta.offset[i] + pColumnInfoData->varmeta.length;
       }
+    }
 
     // copy data
     uint32_t len = pSource->varmeta.length;
@@ -239,7 +243,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co
     if (pColumnInfoData->varmeta.allocLen < len + oldLen) {
       char* tmp = taosMemoryRealloc(pColumnInfoData->pData, len + oldLen);
       if (tmp == NULL) {
-        return TSDB_CODE_VND_OUT_OF_MEMORY;
+        return TSDB_CODE_OUT_OF_MEMORY;
       }
 
       pColumnInfoData->pData = tmp;

source/common/src/trow.c

@@ -220,7 +220,7 @@ static uint8_t tdGetMergedBitmapByte(uint8_t byte) {
 }
 
 /**
- * @brief Merge bitmap from 2 bits to 1 bits, and the memory buffer should be guaranteed by the invoker.
+ * @brief Merge bitmap from 2 bits to 1 bit, and the memory buffer should be guaranteed by the invoker.
  *
  * @param srcBitmap
  * @param nBits

source/common/src/ttypes.c

@@ -651,35 +651,35 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void *buf
   switch (type) {
     case TSDB_DATA_TYPE_INT:
     case TSDB_DATA_TYPE_UINT: {
-      TSWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight), int32_t);
+      TSWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight));
       break;
     }
 
     case TSDB_DATA_TYPE_BIGINT:
     case TSDB_DATA_TYPE_UBIGINT:
     case TSDB_DATA_TYPE_TIMESTAMP: {
-      TSWAP(*(int64_t *)(pLeft), *(int64_t *)(pRight), int64_t);
+      TSWAP(*(int64_t *)(pLeft), *(int64_t *)(pRight));
       break;
     }
     case TSDB_DATA_TYPE_DOUBLE: {
-      TSWAP(*(double *)(pLeft), *(double *)(pRight), double);
+      TSWAP(*(double *)(pLeft), *(double *)(pRight));
       break;
     }
     case TSDB_DATA_TYPE_SMALLINT:
     case TSDB_DATA_TYPE_USMALLINT: {
-      TSWAP(*(int16_t *)(pLeft), *(int16_t *)(pRight), int16_t);
+      TSWAP(*(int16_t *)(pLeft), *(int16_t *)(pRight));
       break;
     }
 
     case TSDB_DATA_TYPE_FLOAT: {
-      TSWAP(*(float *)(pLeft), *(float *)(pRight), float);
+      TSWAP(*(float *)(pLeft), *(float *)(pRight));
       break;
     }
 
     case TSDB_DATA_TYPE_BOOL:
     case TSDB_DATA_TYPE_TINYINT:
     case TSDB_DATA_TYPE_UTINYINT: {
-      TSWAP(*(int8_t *)(pLeft), *(int8_t *)(pRight), int8_t);
+      TSWAP(*(int8_t *)(pLeft), *(int8_t *)(pRight));
       break;
     }
 

source/dnode/mgmt/implement/src/dmEps.c

@@ -120,7 +120,7 @@ int32_t dmReadEps(SDnode *pDnode) {
       goto PRASE_DNODE_OVER;
     }
 
-    dnodeEp.id = dnodeId->valueint;
+    dnodeEp.id = did->valueint;
 
     cJSON *dnodeFqdn = cJSON_GetObjectItem(node, "fqdn");
     if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) {
@@ -156,11 +156,6 @@ PRASE_DNODE_OVER:
   if (root != NULL) cJSON_Delete(root);
   if (pFile != NULL) taosCloseFile(&pFile);
 
-  if (dmIsEpChanged(pDnode, pDnode->data.dnodeId, pDnode->data.localEp)) {
-    dError("localEp %s different with %s and need reconfigured", pDnode->data.localEp, file);
-    return -1;
-  }
-
   if (taosArrayGetSize(pDnode->data.dnodeEps) == 0) {
     SDnodeEp dnodeEp = {0};
     dnodeEp.isMnode = 1;
@@ -170,6 +165,11 @@ PRASE_DNODE_OVER:
 
   dmResetEps(pDnode, pDnode->data.dnodeEps);
 
+  if (dmIsEpChanged(pDnode, pDnode->data.dnodeId, pDnode->data.localEp)) {
+    dError("localEp %s different with %s and need reconfigured", pDnode->data.localEp, file);
+    return -1;
+  }
+
   terrno = code;
   return code;
 }
@@ -291,13 +291,17 @@ static void dmPrintEps(SDnode *pDnode) {
 
 static bool dmIsEpChanged(SDnode *pDnode, int32_t dnodeId, const char *ep) {
   bool changed = false;
+  if (dnodeId == 0) return changed;
   taosRLockLatch(&pDnode->data.latch);
 
   SDnodeEp *pDnodeEp = taosHashGet(pDnode->data.dnodeHash, &dnodeId, sizeof(int32_t));
   if (pDnodeEp != NULL) {
-    char epstr[TSDB_EP_LEN + 1];
+    char epstr[TSDB_EP_LEN + 1] = {0};
     snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port);
-    changed = strcmp(ep, epstr) != 0;
+    changed = (strcmp(ep, epstr) != 0);
+    if (changed) {
+      dError("dnode:%d, localEp %s different from %s", dnodeId, ep, epstr);
+    }
   }
 
   taosRUnLockLatch(&pDnode->data.latch);

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -107,14 +107,11 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
 
   pCfg->vgId = pCreate->vgId;
   strcpy(pCfg->dbname, pCreate->db);
-  pCfg->szBuf = pCreate->cacheBlockSize * 1024 * 1024;
-  pCfg->streamMode = pCreate->streamMode;
   pCfg->isWeak = true;
   pCfg->tsdbCfg.days = 10;
-  pCfg->tsdbCfg.keep2 = 3650;  // pCreate->daysToKeep0;
-  pCfg->tsdbCfg.keep0 = 3650;  // pCreate->daysToKeep2;
-  pCfg->tsdbCfg.keep1 = 3650;  // pCreate->daysToKeep0;
-  pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize;
+  pCfg->tsdbCfg.keep2 = 3650;
+  pCfg->tsdbCfg.keep0 = 3650;
+  pCfg->tsdbCfg.keep1 = 3650;
   pCfg->tsdbCfg.retentions = pCreate->pRetensions;
   pCfg->walCfg.vgId = pCreate->vgId;
   pCfg->hashBegin = pCreate->hashBegin;

source/dnode/mgmt/test/vnode/vnode.cpp

@@ -33,22 +33,18 @@ TEST_F(DndTestVnode, 01_Create_Vnode) {
     strcpy(createReq.db, "1.d1");
     createReq.dbUid = 9527;
     createReq.vgVersion = 1;
-    createReq.cacheBlockSize = 16;
-    createReq.totalBlocks = 10;
     createReq.daysPerFile = 10;
     createReq.daysToKeep0 = 3650;
     createReq.daysToKeep1 = 3650;
     createReq.daysToKeep2 = 3650;
     createReq.minRows = 100;
     createReq.minRows = 4096;
-    createReq.commitTime = 3600;
     createReq.fsyncPeriod = 3000;
     createReq.walLevel = 1;
     createReq.precision = 0;
     createReq.compression = 2;
     createReq.replica = 1;
     createReq.strict = 1;
-    createReq.update = 0;
     createReq.cacheLastRow = 0;
     createReq.selfIndex = 0;
     for (int r = 0; r < createReq.replica; ++r) {
@@ -75,27 +71,15 @@ TEST_F(DndTestVnode, 01_Create_Vnode) {
 TEST_F(DndTestVnode, 02_Alter_Vnode) {
   for (int i = 0; i < 3; ++i) {
     SAlterVnodeReq alterReq = {0};
-    alterReq.vgId = 2;
-    alterReq.dnodeId = 1;
-    strcpy(alterReq.db, "1.d1");
-    alterReq.dbUid = 9527;
     alterReq.vgVersion = 2;
-    alterReq.cacheBlockSize = 16;
-    alterReq.totalBlocks = 10;
     alterReq.daysPerFile = 10;
     alterReq.daysToKeep0 = 3650;
     alterReq.daysToKeep1 = 3650;
     alterReq.daysToKeep2 = 3650;
-    alterReq.minRows = 100;
-    alterReq.minRows = 4096;
-    alterReq.commitTime = 3600;
     alterReq.fsyncPeriod = 3000;
     alterReq.walLevel = 1;
-    alterReq.precision = 0;
-    alterReq.compression = 2;
     alterReq.replica = 1;
     alterReq.strict = 1;
-    alterReq.update = 0;
     alterReq.cacheLastRow = 0;
     alterReq.selfIndex = 0;
     for (int r = 0; r < alterReq.replica; ++r) {

source/dnode/mnode/impl/inc/mndConsumer.h

@@ -23,10 +23,8 @@ extern "C" {
 #endif
 
 enum {
-  // MQ_CONSUMER_STATUS__INIT = 1,
   MQ_CONSUMER_STATUS__MODIFY = 1,
   MQ_CONSUMER_STATUS__MODIFY_IN_REB,
-  // MQ_CONSUMER_STATUS__IDLE,
   MQ_CONSUMER_STATUS__READY,
   MQ_CONSUMER_STATUS__LOST,
   MQ_CONSUMER_STATUS__LOST_IN_REB,
@@ -46,6 +44,11 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw);
 
 int32_t mndSetConsumerCommitLogs(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer);
 
+bool mndRebTryStart();
+void mndRebEnd();
+void mndRebCntInc();
+void mndRebCntDec();
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mnode/impl/inc/mndDef.h

@@ -89,6 +89,7 @@ typedef enum {
   TRN_TYPE_DROP_STREAM = 1020,
   TRN_TYPE_ALTER_STREAM = 1021,
   TRN_TYPE_CONSUMER_LOST = 1022,
+  TRN_TYPE_CONSUMER_RECOVER = 1023,
   TRN_TYPE_BASIC_SCOPE_END,
   TRN_TYPE_GLOBAL_SCOPE = 2000,
   TRN_TYPE_CREATE_DNODE = 2001,
@@ -256,26 +257,23 @@ typedef struct {
 
 typedef struct {
   int32_t numOfVgroups;
-  int32_t cacheBlockSize;
-  int32_t totalBlocks;
+  int32_t numOfStables;
+  int32_t buffer;
+  int32_t pageSize;
+  int32_t pages;
   int32_t daysPerFile;
   int32_t daysToKeep0;
   int32_t daysToKeep1;
   int32_t daysToKeep2;
   int32_t minRows;
   int32_t maxRows;
-  int32_t commitTime;
   int32_t fsyncPeriod;
-  int32_t ttl;
   int8_t  walLevel;
   int8_t  precision;
   int8_t  compression;
   int8_t  replications;
   int8_t  strict;
-  int8_t  update;
   int8_t  cacheLastRow;
-  int8_t  streamMode;
-  int8_t  singleSTable;
   int8_t  hashMethod;  // default is 1
   int32_t numOfRetensions;
   SArray* pRetensions;
@@ -315,7 +313,6 @@ typedef struct {
   int64_t   pointsWritten;
   int8_t    compact;
   int8_t    replica;
-  int8_t    streamMode;
   SVnodeGid vnodeGid[TSDB_MAX_REPLICA];
 } SVgObj;
 
@@ -463,12 +460,14 @@ enum {
   CONSUMER_UPDATE__ADD,
   CONSUMER_UPDATE__REMOVE,
   CONSUMER_UPDATE__LOST,
+  CONSUMER_UPDATE__RECOVER,
   CONSUMER_UPDATE__MODIFY,
 };
 
 typedef struct {
   int64_t consumerId;
   char    cgroup[TSDB_CGROUP_LEN];
+  char    appId[TSDB_CGROUP_LEN];
   int8_t  updateType;  // used only for update
   int32_t epoch;
   int32_t status;
@@ -479,6 +478,17 @@ typedef struct {
   SArray*  currentTopics;     // SArray<char*>
   SArray*  rebNewTopics;      // SArray<char*>
   SArray*  rebRemovedTopics;  // SArray<char*>
+
+  // subscribed by user
+  SArray* assignedTopics;  // SArray<char*>
+
+  // data for display
+  int32_t pid;
+  SEpSet  ep;
+  int64_t upTime;
+  int64_t subscribeTime;
+  int64_t rebalanceTime;
+
 } SMqConsumerObj;
 
 SMqConsumerObj* tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]);
@@ -500,12 +510,12 @@ void*    tDecodeSMqVgEp(const void* buf, SMqVgEp* pVgEp);
 typedef struct {
   int64_t consumerId;  // -1 for unassigned
   SArray* vgs;         // SArray<SMqVgEp*>
-} SMqConsumerEpInSub;
+} SMqConsumerEp;
 
-SMqConsumerEpInSub* tCloneSMqConsumerEpInSub(const SMqConsumerEpInSub* pEpInSub);
-void                tDeleteSMqConsumerEpInSub(SMqConsumerEpInSub* pEpInSub);
-int32_t             tEncodeSMqConsumerEpInSub(void** buf, const SMqConsumerEpInSub* pEpInSub);
-void*               tDecodeSMqConsumerEpInSub(const void* buf, SMqConsumerEpInSub* pEpInSub);
+SMqConsumerEp* tCloneSMqConsumerEp(const SMqConsumerEp* pEp);
+void           tDeleteSMqConsumerEp(SMqConsumerEp* pEp);
+int32_t        tEncodeSMqConsumerEp(void** buf, const SMqConsumerEp* pEp);
+void*          tDecodeSMqConsumerEp(const void* buf, SMqConsumerEp* pEp);
 
 typedef struct {
   char      key[TSDB_SUBSCRIBE_KEY_LEN];
@@ -515,9 +525,8 @@ typedef struct {
   int8_t    withTbName;
   int8_t    withSchema;
   int8_t    withTag;
-  SHashObj* consumerHash;  // consumerId -> SMqConsumerEpInSub
-  // TODO put -1 into unassignVgs
-  // SArray*   unassignedVgs;
+  SHashObj* consumerHash;   // consumerId -> SMqConsumerEp
+  SArray*   unassignedVgs;  // SArray<SMqVgEp*>
 } SMqSubscribeObj;
 
 SMqSubscribeObj* tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]);
@@ -528,7 +537,7 @@ void*            tDecodeSubscribeObj(const void* buf, SMqSubscribeObj* pSub);
 
 typedef struct {
   int32_t epoch;
-  SArray* consumers;  // SArray<SMqConsumerEpInSub*>
+  SArray* consumers;  // SArray<SMqConsumerEp*>
 } SMqSubActionLogEntry;
 
 SMqSubActionLogEntry* tCloneSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry);
@@ -569,8 +578,9 @@ typedef struct {
 
 typedef struct {
   char     name[TSDB_TOPIC_FNAME_LEN];
-  char     db[TSDB_DB_FNAME_LEN];
-  char     outputSTbName[TSDB_TABLE_FNAME_LEN];
+  char     sourceDb[TSDB_DB_FNAME_LEN];
+  char     targetDb[TSDB_DB_FNAME_LEN];
+  char     targetSTbName[TSDB_TABLE_FNAME_LEN];
   int64_t  createTime;
   int64_t  updateTime;
   int64_t  uid;

source/dnode/mnode/impl/inc/mndInfoSchema.h

@@ -22,18 +22,6 @@
 extern "C" {
 #endif
 
-typedef struct SInfosTableSchema {
-  const char   *name;
-  const int32_t type;
-  const int32_t bytes;
-} SInfosTableSchema;
-
-typedef struct SInfosTableMeta {
-  const char              *name;
-  const SInfosTableSchema *schema;
-  const int32_t            colNum;
-} SInfosTableMeta;
-
 int32_t mndInitInfos(SMnode *pMnode);
 void    mndCleanupInfos(SMnode *pMnode);
 int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp);

source/dnode/mnode/impl/inc/mndPerfSchema.h

@@ -22,18 +22,6 @@
 extern "C" {
 #endif
 
-typedef struct SPerfsTableSchema {
-  char   *name;
-  int32_t type;
-  int32_t bytes;
-} SPerfsTableSchema;
-
-typedef struct SPerfsTableMeta {
-  char                    *name;
-  const SPerfsTableSchema *schema;
-  int32_t                  colNum;
-} SPerfsTableMeta;
-
 int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp);
 int32_t mndInitPerfs(SMnode *pMnode);
 void mndCleanupPerfs(SMnode *pMnode);

source/dnode/mnode/impl/inc/mndShow.h

@@ -17,6 +17,7 @@
 #define _TD_MND_SHOW_H_
 
 #include "mndInt.h"
+#include "systable.h"
 
 #ifdef __cplusplus
 extern "C" {

source/dnode/mnode/impl/inc/mndStb.h

@@ -29,6 +29,7 @@ void     mndReleaseStb(SMnode *pMnode, SStbObj *pStb);
 SSdbRaw *mndStbActionEncode(SStbObj *pStb);
 int32_t  mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbs, int32_t numOfStbs, void **ppRsp,
                             int32_t *pRspLen);
+int32_t  mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/inc/mndTrans.h

@@ -36,8 +36,8 @@ typedef struct {
 typedef enum {
   TEST_TRANS_START_FUNC = 1,
   TEST_TRANS_STOP_FUNC = 2,
-  CONSUME_TRANS_START_FUNC = 3,
-  CONSUME_TRANS_STOP_FUNC = 4,
+  MQ_REB_TRANS_START_FUNC = 3,
+  MQ_REB_TRANS_STOP_FUNC = 4,
 } ETrnFuncType;
 
 typedef void (*TransCbFp)(SMnode *pMnode, void *param, int32_t paramLen);

source/dnode/mnode/impl/src/mndConsumer.c

@@ -35,7 +35,9 @@
 
 #define MND_CONSUMER_LOST_HB_CNT 3
 
-static int8_t mqInRebFlag = 0;
+static int8_t mqRebLock = 0;
+
+static const char *mndConsumerStatusName(int status);
 
 static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer);
 static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer);
@@ -48,6 +50,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg);
 static int32_t mndProcessAskEpReq(SNodeMsg *pMsg);
 static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg);
 static int32_t mndProcessConsumerLostMsg(SNodeMsg *pMsg);
+static int32_t mndProcessConsumerRecoverMsg(SNodeMsg *pMsg);
 
 int32_t mndInitConsumer(SMnode *pMnode) {
   SSdbTable table = {.sdbType = SDB_CONSUMER,
@@ -62,11 +65,27 @@ int32_t mndInitConsumer(SMnode *pMnode) {
   mndSetMsgHandle(pMnode, TDMT_MND_MQ_ASK_EP, mndProcessAskEpReq);
   mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg);
   mndSetMsgHandle(pMnode, TDMT_MND_MQ_CONSUMER_LOST, mndProcessConsumerLostMsg);
+  mndSetMsgHandle(pMnode, TDMT_MND_MQ_CONSUMER_RECOVER, mndProcessConsumerRecoverMsg);
+
+  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONSUMERS, mndRetrieveConsumer);
+  mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONSUMERS, mndCancelGetNextConsumer);
+
   return sdbSetTable(pMnode->pSdb, table);
 }
 
 void mndCleanupConsumer(SMnode *pMnode) {}
 
+bool mndRebTryStart() {
+  int8_t old = atomic_val_compare_exchange_8(&mqRebLock, 0, 1);
+  return old == 0;
+}
+
+void mndRebEnd() { atomic_sub_fetch_8(&mqRebLock, 1); }
+
+void mndRebCntInc() { atomic_add_fetch_8(&mqRebLock, 1); }
+
+void mndRebCntDec() { atomic_sub_fetch_8(&mqRebLock, 1); }
+
 static int32_t mndProcessConsumerLostMsg(SNodeMsg *pMsg) {
   SMnode             *pMnode = pMsg->pNode;
   SMqConsumerLostMsg *pLostMsg = pMsg->rpcMsg.pCont;
@@ -91,6 +110,30 @@ FAIL:
   return -1;
 }
 
+static int32_t mndProcessConsumerRecoverMsg(SNodeMsg *pMsg) {
+  SMnode                *pMnode = pMsg->pNode;
+  SMqConsumerRecoverMsg *pRecoverMsg = pMsg->rpcMsg.pCont;
+  SMqConsumerObj        *pConsumer = mndAcquireConsumer(pMnode, pRecoverMsg->consumerId);
+  ASSERT(pConsumer);
+
+  SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup);
+  pConsumerNew->updateType = CONSUMER_UPDATE__RECOVER;
+
+  mndReleaseConsumer(pMnode, pConsumer);
+
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CONSUMER_RECOVER, &pMsg->rpcMsg);
+  if (pTrans == NULL) goto FAIL;
+  if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto FAIL;
+  if (mndTransPrepare(pMnode, pTrans) != 0) goto FAIL;
+
+  mndTransDrop(pTrans);
+  return 0;
+FAIL:
+  tDeleteSMqConsumerObj(pConsumerNew);
+  mndTransDrop(pTrans);
+  return -1;
+}
+
 static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
   SMqRebSubscribe *pRebSub = taosHashGet(pHash, key, strlen(key) + 1);
   if (pRebSub == NULL) {
@@ -111,8 +154,7 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
   void           *pIter = NULL;
 
   // rebalance cannot be parallel
-  int8_t old = atomic_val_compare_exchange_8(&mqInRebFlag, 0, 1);
-  if (old != 0) {
+  if (!mndRebTryStart()) {
     mInfo("mq rebalance already in progress, do nothing");
     return 0;
   }
@@ -120,7 +162,6 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
   SMqDoRebalanceMsg *pRebMsg = rpcMallocCont(sizeof(SMqDoRebalanceMsg));
   pRebMsg->rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK);
   // TODO set cleanfp
-  pRebMsg->mqInReb = &mqInRebFlag;
 
   // iterate all consumers, find all modification
   while (1) {
@@ -191,7 +232,7 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
     taosHashCleanup(pRebMsg->rebSubHash);
     rpcFreeCont(pRebMsg);
     mTrace("mq rebalance finished, no modification");
-    atomic_store_8(&mqInRebFlag, 0);
+    mndRebEnd();
   }
   return 0;
 }
@@ -216,8 +257,15 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
   // 1. check consumer status
   int32_t status = atomic_load_32(&pConsumer->status);
 
-  if (status == MQ_CONSUMER_STATUS__LOST) {
-    // TODO: recover consumer
+  if (status == MQ_CONSUMER_STATUS__LOST_REBD) {
+    SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg));
+
+    pRecoverMsg->consumerId = consumerId;
+    SRpcMsg *pRpcMsg = taosMemoryCalloc(1, sizeof(SRpcMsg));
+    pRpcMsg->msgType = TDMT_MND_MQ_CONSUMER_RECOVER;
+    pRpcMsg->pCont = pRecoverMsg;
+    pRpcMsg->contLen = sizeof(SMqConsumerRecoverMsg);
+    tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, pRpcMsg);
   }
 
   if (status != MQ_CONSUMER_STATUS__READY) {
@@ -263,8 +311,8 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
       mndReleaseTopic(pMnode, pTopic);
 
       // 2.2 iterate all vg assigned to the consumer of that topic
-      SMqConsumerEpInSub *pEpInSub = taosHashGet(pSub->consumerHash, &consumerId, sizeof(int64_t));
-      int32_t             vgNum = taosArrayGetSize(pEpInSub->vgs);
+      SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &consumerId, sizeof(int64_t));
+      int32_t        vgNum = taosArrayGetSize(pConsumerEp->vgs);
 
       topicEp.vgs = taosArrayInit(vgNum, sizeof(SMqSubVgEp));
       if (topicEp.vgs == NULL) {
@@ -274,7 +322,7 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
       }
 
       for (int32_t j = 0; j < vgNum; j++) {
-        SMqVgEp *pVgEp = taosArrayGetP(pEpInSub->vgs, j);
+        SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j);
         char     offsetKey[TSDB_PARTITION_KEY_LEN];
         mndMakePartitionKey(offsetKey, pConsumer->cgroup, topic, pVgEp->vgId);
         // 2.2.1 build vg ep
@@ -366,10 +414,14 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
   if (pConsumerOld == NULL) {
     pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup);
     pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY;
-    /*pConsumerNew->waitingRebTopics = newSub;*/
     pConsumerNew->rebNewTopics = newSub;
     subscribe.topicNames = NULL;
 
+    for (int32_t i = 0; i < newTopicNum; i++) {
+      char *newTopicCopy = strdup(taosArrayGetP(newSub, i));
+      taosArrayPush(pConsumerNew->assignedTopics, &newTopicCopy);
+    }
+
     STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, &pMsg->rpcMsg);
     if (pTrans == NULL) goto SUBSCRIBE_OVER;
     if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto SUBSCRIBE_OVER;
@@ -389,7 +441,11 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
       goto SUBSCRIBE_OVER;
     }
     pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY;
-    /*pConsumerOld->waitingRebTopics = newSub;*/
+
+    for (int32_t i = 0; i < newTopicNum; i++) {
+      char *newTopicCopy = strdup(taosArrayGetP(newSub, i));
+      taosArrayPush(pConsumerNew->assignedTopics, &newTopicCopy);
+    }
 
     int32_t oldTopicNum = 0;
     if (pConsumerOld->currentTopics) {
@@ -532,6 +588,7 @@ CM_DECODE_OVER:
 
 static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) {
   mTrace("consumer:%" PRId64 ", perform insert action", pConsumer->consumerId);
+  pConsumer->subscribeTime = pConsumer->upTime;
   return 0;
 }
 
@@ -557,17 +614,45 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
     pOldConsumer->rebRemovedTopics = pNewConsumer->rebRemovedTopics;
     pNewConsumer->rebRemovedTopics = tmp;
 
+    tmp = pOldConsumer->assignedTopics;
+    pOldConsumer->assignedTopics = pNewConsumer->assignedTopics;
+    pNewConsumer->assignedTopics = tmp;
+
+    pOldConsumer->subscribeTime = pNewConsumer->upTime;
+
     pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY;
   } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) {
+    ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);
+    ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);
+
     int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics);
-    pOldConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *));
+    /*pOldConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *));*/
     for (int32_t i = 0; i < sz; i++) {
       char *topic = strdup(taosArrayGetP(pOldConsumer->currentTopics, i));
-      taosArrayPush(pNewConsumer->rebRemovedTopics, &topic);
+      taosArrayPush(pOldConsumer->rebRemovedTopics, &topic);
     }
+
+    pOldConsumer->rebalanceTime = pNewConsumer->upTime;
+
     pOldConsumer->status = MQ_CONSUMER_STATUS__LOST;
+  } else if (pNewConsumer->updateType == CONSUMER_UPDATE__RECOVER) {
+    ASSERT(taosArrayGetSize(pOldConsumer->currentTopics) == 0);
+    ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);
+
+    int32_t sz = taosArrayGetSize(pOldConsumer->assignedTopics);
+    for (int32_t i = 0; i < sz; i++) {
+      char *topic = strdup(taosArrayGetP(pOldConsumer->assignedTopics, i));
+      taosArrayPush(pOldConsumer->rebNewTopics, &topic);
+    }
+
+    pOldConsumer->rebalanceTime = pNewConsumer->upTime;
+
+    pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY;
   } else if (pNewConsumer->updateType == CONSUMER_UPDATE__TOUCH) {
     atomic_add_fetch_32(&pOldConsumer->epoch, 1);
+
+    pOldConsumer->rebalanceTime = pNewConsumer->upTime;
+
   } else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) {
     ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1);
     ASSERT(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0);
@@ -612,6 +697,9 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
         pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB;
       }
     }
+
+    pOldConsumer->rebalanceTime = pNewConsumer->upTime;
+
     atomic_add_fetch_32(&pOldConsumer->epoch, 1);
   } else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) {
     ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0);
@@ -668,6 +756,9 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
         pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB;
       }
     }
+
+    pOldConsumer->rebalanceTime = pNewConsumer->upTime;
+
     atomic_add_fetch_32(&pOldConsumer->epoch, 1);
   }
 
@@ -688,3 +779,105 @@ void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer) {
   SSdb *pSdb = pMnode->pSdb;
   sdbRelease(pSdb, pConsumer);
 }
+
+static int32_t mndRetrieveConsumer(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) {
+  SMnode         *pMnode = pReq->pNode;
+  SSdb           *pSdb = pMnode->pSdb;
+  int32_t         numOfRows = 0;
+  SMqConsumerObj *pConsumer = NULL;
+
+  while (numOfRows < rowsCapacity) {
+    pShow->pIter = sdbFetch(pSdb, SDB_CONSUMER, pShow->pIter, (void **)&pConsumer);
+    if (pShow->pIter == NULL) break;
+
+    SColumnInfoData *pColInfo;
+    int32_t          cols = 0;
+
+    taosRLockLatch(&pConsumer->lock);
+
+    // consumer id
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->consumerId, false);
+
+    // group id
+    char groupId[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0};
+    tstrncpy(varDataVal(groupId), pConsumer->cgroup, TSDB_CGROUP_LEN);
+    varDataSetLen(groupId, strlen(varDataVal(groupId)));
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)groupId, false);
+
+    // app id
+    char appId[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0};
+    tstrncpy(varDataVal(appId), pConsumer->appId, TSDB_CGROUP_LEN);
+    varDataSetLen(appId, strlen(varDataVal(appId)));
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)appId, false);
+
+    // status
+    char status[20 + VARSTR_HEADER_SIZE] = {0};
+    tstrncpy(varDataVal(status), mndConsumerStatusName(pConsumer->status), 20);
+    varDataSetLen(status, strlen(varDataVal(status)));
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)status, false);
+
+    // subscribed topics
+    // TODO: split into multiple rows
+    char  topics[TSDB_SHOW_LIST_LEN + VARSTR_HEADER_SIZE] = {0};
+    char *showStr = taosShowStrArray(pConsumer->assignedTopics);
+    tstrncpy(varDataVal(topics), showStr, TSDB_SHOW_LIST_LEN);
+    taosMemoryFree(showStr);
+    varDataSetLen(topics, strlen(varDataVal(topics)));
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)topics, false);
+
+    // pid
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->pid, true);
+
+    // end point
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->ep, true);
+
+    // up time
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->upTime, false);
+
+    // subscribe time
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->subscribeTime, false);
+
+    // rebalance time
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->rebalanceTime, pConsumer->rebalanceTime == 0);
+
+    taosRUnLockLatch(&pConsumer->lock);
+    sdbRelease(pSdb, pConsumer);
+
+    numOfRows++;
+  }
+
+  pShow->numOfRows += numOfRows;
+  return numOfRows;
+}
+
+static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) {
+  SSdb *pSdb = pMnode->pSdb;
+  sdbCancelFetch(pSdb, pIter);
+}
+
+static const char *mndConsumerStatusName(int status) {
+  switch (status) {
+    case MQ_CONSUMER_STATUS__READY:
+      return "ready";
+    case MQ_CONSUMER_STATUS__LOST:
+    case MQ_CONSUMER_STATUS__LOST_REBD:
+    case MQ_CONSUMER_STATUS__LOST_IN_REB:
+      return "lost";
+    case MQ_CONSUMER_STATUS__MODIFY:
+    case MQ_CONSUMER_STATUS__MODIFY_IN_REB:
+      return "rebalancing";
+    default:
+      return "unknown";
+  }
+}

source/dnode/mnode/impl/src/mndDb.c

@@ -23,6 +23,7 @@
 #include "mndTrans.h"
 #include "mndUser.h"
 #include "mndVgroup.h"
+#include "systable.h"
 
 #define DB_VER_NUMBER   1
 #define DB_RESERVE_SIZE 64
@@ -84,26 +85,23 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
   SDB_SET_INT32(pRaw, dataPos, pDb->cfgVersion, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pDb->vgVersion, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfVgroups, _OVER)
-  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.cacheBlockSize, _OVER)
-  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.totalBlocks, _OVER)
+  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfStables, _OVER)
+  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.buffer, _OVER)
+  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.pageSize, _OVER)
+  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.pages, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysPerFile, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep0, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep1, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep2, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.minRows, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.maxRows, _OVER)
-  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.commitTime, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.fsyncPeriod, _OVER)
-  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.ttl, _OVER)
   SDB_SET_INT8(pRaw, dataPos, pDb->cfg.walLevel, _OVER)
   SDB_SET_INT8(pRaw, dataPos, pDb->cfg.precision, _OVER)
   SDB_SET_INT8(pRaw, dataPos, pDb->cfg.compression, _OVER)
   SDB_SET_INT8(pRaw, dataPos, pDb->cfg.replications, _OVER)
   SDB_SET_INT8(pRaw, dataPos, pDb->cfg.strict, _OVER)
-  SDB_SET_INT8(pRaw, dataPos, pDb->cfg.update, _OVER)
   SDB_SET_INT8(pRaw, dataPos, pDb->cfg.cacheLastRow, _OVER)
-  SDB_SET_INT8(pRaw, dataPos, pDb->cfg.streamMode, _OVER)
-  SDB_SET_INT8(pRaw, dataPos, pDb->cfg.singleSTable, _OVER)
   SDB_SET_INT8(pRaw, dataPos, pDb->cfg.hashMethod, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfRetensions, _OVER)
   for (int32_t i = 0; i < pDb->cfg.numOfRetensions; ++i) {
@@ -158,26 +156,23 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
   SDB_GET_INT32(pRaw, dataPos, &pDb->cfgVersion, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pDb->vgVersion, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.numOfVgroups, _OVER)
-  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.cacheBlockSize, _OVER)
-  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.totalBlocks, _OVER)
+  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.numOfStables, _OVER)
+  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.buffer, _OVER)
+  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.pageSize, _OVER)
+  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.pages, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysPerFile, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysToKeep0, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysToKeep1, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.daysToKeep2, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.minRows, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.maxRows, _OVER)
-  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.commitTime, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.fsyncPeriod, _OVER)
-  SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.ttl, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.walLevel, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.precision, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.compression, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.replications, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.strict, _OVER)
-  SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.update, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.cacheLastRow, _OVER)
-  SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.streamMode, _OVER)
-  SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.singleSTable, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.hashMethod, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.numOfRetensions, _OVER)
   if (pDb->cfg.numOfRetensions > 0) {
@@ -268,8 +263,10 @@ static int32_t mndCheckDbName(const char *dbName, SUserObj *pUser) {
 
 static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
   if (pCfg->numOfVgroups < TSDB_MIN_VNODES_PER_DB || pCfg->numOfVgroups > TSDB_MAX_VNODES_PER_DB) return -1;
-  if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) return -1;
-  if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) return -1;
+  if (pCfg->numOfStables < TSDB_DB_STREAM_MODE_OFF || pCfg->numOfStables > TSDB_DB_STREAM_MODE_ON) return -1;
+  if (pCfg->buffer < TSDB_MIN_BUFFER_PER_VNODE || pCfg->buffer > TSDB_MAX_BUFFER_PER_VNODE) return -1;
+  if (pCfg->pageSize < TSDB_MIN_PAGESIZE_PER_VNODE || pCfg->pageSize > TSDB_MAX_PAGESIZE_PER_VNODE) return -1;
+  if (pCfg->pages < TSDB_MIN_PAGES_PER_VNODE || pCfg->pages > TSDB_MAX_PAGES_PER_VNODE) return -1;
   if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1;
   if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP) return -1;
   if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP) return -1;
@@ -280,9 +277,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
   if (pCfg->minRows < TSDB_MIN_MINROWS_FBLOCK || pCfg->minRows > TSDB_MAX_MINROWS_FBLOCK) return -1;
   if (pCfg->maxRows < TSDB_MIN_MAXROWS_FBLOCK || pCfg->maxRows > TSDB_MAX_MAXROWS_FBLOCK) return -1;
   if (pCfg->minRows > pCfg->maxRows) return -1;
-  if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) return -1;
   if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) return -1;
-  if (pCfg->ttl < TSDB_MIN_DB_TTL) return -1;
   if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) return -1;
   if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) return -1;
   if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) return -1;
@@ -290,36 +285,30 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
   if (pCfg->replications > mndGetDnodeSize(pMnode)) return -1;
   if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1;
   if (pCfg->strict > pCfg->replications) return -1;
-  if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) return -1;
   if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1;
-  if (pCfg->streamMode < TSDB_DB_STREAM_MODE_OFF || pCfg->streamMode > TSDB_DB_STREAM_MODE_ON) return -1;
-  if (pCfg->singleSTable < TSDB_DB_SINGLE_STABLE_ON || pCfg->streamMode > TSDB_DB_SINGLE_STABLE_OFF) return -1;
   if (pCfg->hashMethod != 1) return -1;
   return TSDB_CODE_SUCCESS;
 }
 
 static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
   if (pCfg->numOfVgroups < 0) pCfg->numOfVgroups = TSDB_DEFAULT_VN_PER_DB;
-  if (pCfg->cacheBlockSize < 0) pCfg->cacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
-  if (pCfg->totalBlocks < 0) pCfg->totalBlocks = TSDB_DEFAULT_TOTAL_BLOCKS;
-  if (pCfg->daysPerFile < 0) pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE;
+  if (pCfg->numOfStables < 0) pCfg->numOfStables = TSDB_DEFAULT_DB_SINGLE_STABLE;
+  if (pCfg->buffer < 0) pCfg->buffer = TSDB_DEFAULT_BUFFER_PER_VNODE;
+  if (pCfg->pageSize < 0) pCfg->pageSize = TSDB_DEFAULT_PAGES_PER_VNODE;
+  if (pCfg->pages < 0) pCfg->pages = TSDB_MAX_PAGESIZE_PER_VNODE;
+  if (pCfg->daysPerFile < 0) pCfg->daysPerFile = TSDB_DEFAULT_DURATION_PER_FILE;
   if (pCfg->daysToKeep0 < 0) pCfg->daysToKeep0 = TSDB_DEFAULT_KEEP;
   if (pCfg->daysToKeep1 < 0) pCfg->daysToKeep1 = pCfg->daysToKeep0;
   if (pCfg->daysToKeep2 < 0) pCfg->daysToKeep2 = pCfg->daysToKeep1;
   if (pCfg->minRows < 0) pCfg->minRows = TSDB_DEFAULT_MINROWS_FBLOCK;
   if (pCfg->maxRows < 0) pCfg->maxRows = TSDB_DEFAULT_MAXROWS_FBLOCK;
-  if (pCfg->commitTime < 0) pCfg->commitTime = TSDB_DEFAULT_COMMIT_TIME;
   if (pCfg->fsyncPeriod < 0) pCfg->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
-  if (pCfg->ttl < 0) pCfg->ttl = TSDB_DEFAULT_DB_TTL;
   if (pCfg->walLevel < 0) pCfg->walLevel = TSDB_DEFAULT_WAL_LEVEL;
   if (pCfg->precision < 0) pCfg->precision = TSDB_DEFAULT_PRECISION;
   if (pCfg->compression < 0) pCfg->compression = TSDB_DEFAULT_COMP_LEVEL;
   if (pCfg->replications < 0) pCfg->replications = TSDB_DEFAULT_DB_REPLICA;
   if (pCfg->strict < 0) pCfg->strict = TSDB_DEFAULT_DB_STRICT;
-  if (pCfg->update < 0) pCfg->update = TSDB_DEFAULT_DB_UPDATE;
   if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW;
-  if (pCfg->streamMode < 0) pCfg->streamMode = TSDB_DEFAULT_DB_STREAM_MODE;
-  if (pCfg->singleSTable < 0) pCfg->singleSTable = TSDB_DEFAULT_DB_SINGLE_STABLE;
   if (pCfg->numOfRetensions < 0) pCfg->numOfRetensions = 0;
 }
 
@@ -445,26 +434,23 @@ static int32_t mndCreateDb(SMnode *pMnode, SNodeMsg *pReq, SCreateDbReq *pCreate
   memcpy(dbObj.createUser, pUser->user, TSDB_USER_LEN);
   dbObj.cfg = (SDbCfg){
       .numOfVgroups = pCreate->numOfVgroups,
-      .cacheBlockSize = pCreate->cacheBlockSize,
-      .totalBlocks = pCreate->totalBlocks,
+      .numOfStables = pCreate->numOfStables,
+      .buffer = pCreate->buffer,
+      .pageSize = pCreate->pageSize,
+      .pages = pCreate->pages,
       .daysPerFile = pCreate->daysPerFile,
       .daysToKeep0 = pCreate->daysToKeep0,
       .daysToKeep1 = pCreate->daysToKeep1,
       .daysToKeep2 = pCreate->daysToKeep2,
       .minRows = pCreate->minRows,
       .maxRows = pCreate->maxRows,
-      .commitTime = pCreate->commitTime,
       .fsyncPeriod = pCreate->fsyncPeriod,
-      .ttl = pCreate->ttl,
       .walLevel = pCreate->walLevel,
       .precision = pCreate->precision,
       .compression = pCreate->compression,
       .replications = pCreate->replications,
       .strict = pCreate->strict,
-      .update = pCreate->update,
       .cacheLastRow = pCreate->cacheLastRow,
-      .streamMode = pCreate->streamMode,
-      .singleSTable = pCreate->singleSTable,
       .hashMethod = 1,
   };
 
@@ -568,8 +554,23 @@ _OVER:
 static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
   terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED;
 
-  if (pAlter->totalBlocks >= 0 && pAlter->totalBlocks != pDb->cfg.totalBlocks) {
-    pDb->cfg.totalBlocks = pAlter->totalBlocks;
+  if (pAlter->buffer >= 0 && pAlter->buffer != pDb->cfg.buffer) {
+    pDb->cfg.buffer = pAlter->buffer;
+    terrno = 0;
+  }
+
+  if (pAlter->pages >= 0 && pAlter->pages != pDb->cfg.pages) {
+    pDb->cfg.pages = pAlter->pages;
+    terrno = 0;
+  }
+
+  if (pAlter->pageSize >= 0 && pAlter->pageSize != pDb->cfg.pageSize) {
+    pDb->cfg.pageSize = pAlter->pageSize;
+    terrno = 0;
+  }
+
+  if (pAlter->daysPerFile >= 0 && pAlter->daysPerFile != pDb->cfg.daysPerFile) {
+    pDb->cfg.daysPerFile = pAlter->daysPerFile;
     terrno = 0;
   }
 
@@ -637,10 +638,14 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p
 void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) {
   SAlterVnodeReq alterReq = {0};
   alterReq.vgVersion = pVgroup->version;
-  alterReq.totalBlocks = pDb->cfg.totalBlocks;
+  alterReq.buffer = pDb->cfg.buffer;
+  alterReq.pages = pDb->cfg.pages;
+  alterReq.pageSize = pDb->cfg.pageSize;
+  alterReq.daysPerFile = pDb->cfg.daysPerFile;
   alterReq.daysToKeep0 = pDb->cfg.daysToKeep0;
   alterReq.daysToKeep1 = pDb->cfg.daysToKeep1;
   alterReq.daysToKeep2 = pDb->cfg.daysToKeep2;
+  alterReq.fsyncPeriod = pDb->cfg.fsyncPeriod;
   alterReq.walLevel = pDb->cfg.walLevel;
   alterReq.strict = pDb->cfg.strict;
   alterReq.cacheLastRow = pDb->cfg.cacheLastRow;
@@ -833,26 +838,23 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) {
   }
 
   cfgRsp.numOfVgroups = pDb->cfg.numOfVgroups;
-  cfgRsp.cacheBlockSize = pDb->cfg.cacheBlockSize;
-  cfgRsp.totalBlocks = pDb->cfg.totalBlocks;
+  cfgRsp.numOfStables = pDb->cfg.numOfStables;
+  cfgRsp.buffer = pDb->cfg.buffer;
+  cfgRsp.pageSize = pDb->cfg.pageSize;
+  cfgRsp.pages = pDb->cfg.pages;
   cfgRsp.daysPerFile = pDb->cfg.daysPerFile;
   cfgRsp.daysToKeep0 = pDb->cfg.daysToKeep0;
   cfgRsp.daysToKeep1 = pDb->cfg.daysToKeep1;
   cfgRsp.daysToKeep2 = pDb->cfg.daysToKeep2;
   cfgRsp.minRows = pDb->cfg.minRows;
   cfgRsp.maxRows = pDb->cfg.maxRows;
-  cfgRsp.commitTime = pDb->cfg.commitTime;
   cfgRsp.fsyncPeriod = pDb->cfg.fsyncPeriod;
-  cfgRsp.ttl = pDb->cfg.ttl;
   cfgRsp.walLevel = pDb->cfg.walLevel;
   cfgRsp.precision = pDb->cfg.precision;
   cfgRsp.compression = pDb->cfg.compression;
   cfgRsp.replications = pDb->cfg.replications;
   cfgRsp.strict = pDb->cfg.strict;
-  cfgRsp.update = pDb->cfg.update;
   cfgRsp.cacheLastRow = pDb->cfg.cacheLastRow;
-  cfgRsp.streamMode = pDb->cfg.streamMode;
-  cfgRsp.singleSTable = pDb->cfg.singleSTable;
   cfgRsp.numOfRetensions = pDb->cfg.numOfRetensions;
   cfgRsp.pRetensions = pDb->cfg.pRetensions;
 
@@ -1191,7 +1193,7 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) {
   }
 
   char *p = strchr(usedbReq.db, '.');
-  if (p && 0 == strcmp(p + 1, TSDB_INFORMATION_SCHEMA_DB)) {
+  if (p && ((0 == strcmp(p + 1, TSDB_INFORMATION_SCHEMA_DB) || (0 == strcmp(p + 1, TSDB_PERFORMANCE_SCHEMA_DB))))) {
     memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN);
     int32_t vgVersion = mndGetGlobalVgroupVersion(pMnode);
     if (usedbReq.vgVersion < vgVersion) {
@@ -1428,7 +1430,6 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in
     colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.replications, false);
 
     const char *src = pDb->cfg.strict ? "strict" : "nostrict";
-    char        b[9 + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_SIZE_TO_VARSTR(b, src, strlen(src));
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, rows, (const char *)b, false);
@@ -1451,10 +1452,13 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in
     colDataAppend(pColInfo, rows, (const char *)tmp, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.cacheBlockSize, false);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.buffer, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.pageSize, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.totalBlocks, false);
+    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.pages, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.minRows, false);
@@ -1495,14 +1499,10 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, rows, (const char *)t, false);
 
+    //  single stable model
+    int8_t m = 0;
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.ttl, false);
-
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.singleSTable, false);
-
-    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.streamMode, false);
+    colDataAppend(pColInfo, rows, (const char *)&m, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
     colDataAppend(pColInfo, rows, (const char *)b, false);
@@ -1520,7 +1520,6 @@ static void setInformationSchemaDbCfg(SDbObj *pDbObj) {
   pDbObj->cfg.numOfVgroups = 0;
   pDbObj->cfg.strict = 1;
   pDbObj->cfg.replications = 1;
-  pDbObj->cfg.update = 1;
   pDbObj->cfg.precision = TSDB_TIME_PRECISION_MILLI;
 }
 
@@ -1532,7 +1531,6 @@ static void setPerfSchemaDbCfg(SDbObj *pDbObj) {
   pDbObj->cfg.numOfVgroups = 0;
   pDbObj->cfg.strict = 1;
   pDbObj->cfg.replications = 1;
-  pDbObj->cfg.update = 1;
   pDbObj->cfg.precision = TSDB_TIME_PRECISION_MILLI;
 }
 

source/dnode/mnode/impl/src/mndDef.c

@@ -34,15 +34,20 @@ SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_L
   pConsumer->currentTopics = taosArrayInit(0, sizeof(void *));
   pConsumer->rebNewTopics = taosArrayInit(0, sizeof(void *));
   pConsumer->rebRemovedTopics = taosArrayInit(0, sizeof(void *));
+  pConsumer->assignedTopics = taosArrayInit(0, sizeof(void *));
 
-  if (pConsumer->currentTopics == NULL || pConsumer->rebNewTopics == NULL || pConsumer->rebRemovedTopics == NULL) {
+  if (pConsumer->currentTopics == NULL || pConsumer->rebNewTopics == NULL || pConsumer->rebRemovedTopics == NULL ||
+      pConsumer->assignedTopics == NULL) {
     taosArrayDestroy(pConsumer->currentTopics);
     taosArrayDestroy(pConsumer->rebNewTopics);
     taosArrayDestroy(pConsumer->rebRemovedTopics);
+    taosArrayDestroy(pConsumer->assignedTopics);
     taosMemoryFree(pConsumer);
     return NULL;
   }
 
+  pConsumer->upTime = taosGetTimestampMs();
+
   return pConsumer;
 }
 
@@ -56,6 +61,9 @@ void tDeleteSMqConsumerObj(SMqConsumerObj *pConsumer) {
   if (pConsumer->rebRemovedTopics) {
     taosArrayDestroyP(pConsumer->rebRemovedTopics, (FDelete)taosMemoryFree);
   }
+  if (pConsumer->assignedTopics) {
+    taosArrayDestroyP(pConsumer->assignedTopics, (FDelete)taosMemoryFree);
+  }
 }
 
 int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) {
@@ -67,6 +75,12 @@ int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) {
   tlen += taosEncodeFixedI32(buf, pConsumer->epoch);
   tlen += taosEncodeFixedI32(buf, pConsumer->status);
 
+  tlen += taosEncodeFixedI32(buf, pConsumer->pid);
+  tlen += taosEncodeSEpSet(buf, &pConsumer->ep);
+  tlen += taosEncodeFixedI64(buf, pConsumer->upTime);
+  tlen += taosEncodeFixedI64(buf, pConsumer->subscribeTime);
+  tlen += taosEncodeFixedI64(buf, pConsumer->rebalanceTime);
+
   // current topics
   if (pConsumer->currentTopics) {
     sz = taosArrayGetSize(pConsumer->currentTopics);
@@ -103,6 +117,18 @@ int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) {
     tlen += taosEncodeFixedI32(buf, 0);
   }
 
+  // lost topics
+  if (pConsumer->assignedTopics) {
+    sz = taosArrayGetSize(pConsumer->assignedTopics);
+    tlen += taosEncodeFixedI32(buf, sz);
+    for (int32_t i = 0; i < sz; i++) {
+      char *topic = taosArrayGetP(pConsumer->assignedTopics, i);
+      tlen += taosEncodeString(buf, topic);
+    }
+  } else {
+    tlen += taosEncodeFixedI32(buf, 0);
+  }
+
   return tlen;
 }
 
@@ -114,6 +140,12 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) {
   buf = taosDecodeFixedI32(buf, &pConsumer->epoch);
   buf = taosDecodeFixedI32(buf, &pConsumer->status);
 
+  buf = taosDecodeFixedI32(buf, &pConsumer->pid);
+  buf = taosDecodeSEpSet(buf, &pConsumer->ep);
+  buf = taosDecodeFixedI64(buf, &pConsumer->upTime);
+  buf = taosDecodeFixedI64(buf, &pConsumer->subscribeTime);
+  buf = taosDecodeFixedI64(buf, &pConsumer->rebalanceTime);
+
   // current topics
   buf = taosDecodeFixedI32(buf, &sz);
   pConsumer->currentTopics = taosArrayInit(sz, sizeof(void *));
@@ -141,6 +173,15 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) {
     taosArrayPush(pConsumer->rebRemovedTopics, &topic);
   }
 
+  // reb removed topics
+  buf = taosDecodeFixedI32(buf, &sz);
+  pConsumer->assignedTopics = taosArrayInit(sz, sizeof(void *));
+  for (int32_t i = 0; i < sz; i++) {
+    char *topic;
+    buf = taosDecodeString(buf, &topic);
+    taosArrayPush(pConsumer->assignedTopics, &topic);
+  }
+
   return (void *)buf;
 }
 
@@ -170,42 +211,47 @@ void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp) {
   return (void *)buf;
 }
 
-SMqConsumerEpInSub *tCloneSMqConsumerEpInSub(const SMqConsumerEpInSub *pEpInSub) {
-  SMqConsumerEpInSub *pEpInSubNew = taosMemoryMalloc(sizeof(SMqConsumerEpInSub));
-  if (pEpInSubNew == NULL) return NULL;
-  pEpInSubNew->consumerId = pEpInSub->consumerId;
-  pEpInSubNew->vgs = taosArrayDeepCopy(pEpInSub->vgs, (FCopy)tCloneSMqVgEp);
-  return pEpInSubNew;
+SMqConsumerEp *tCloneSMqConsumerEp(const SMqConsumerEp *pConsumerEpOld) {
+  SMqConsumerEp *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEp));
+  if (pConsumerEpNew == NULL) return NULL;
+  pConsumerEpNew->consumerId = pConsumerEpOld->consumerId;
+  pConsumerEpNew->vgs = taosArrayDeepCopy(pConsumerEpOld->vgs, (FCopy)tCloneSMqVgEp);
+  return pConsumerEpNew;
 }
 
-void tDeleteSMqConsumerEpInSub(SMqConsumerEpInSub *pEpInSub) {
-  taosArrayDestroyEx(pEpInSub->vgs, (FDelete)tDeleteSMqVgEp);
+void tDeleteSMqConsumerEp(SMqConsumerEp *pConsumerEp) {
+  //
+  taosArrayDestroyP(pConsumerEp->vgs, (FDelete)tDeleteSMqVgEp);
 }
 
-int32_t tEncodeSMqConsumerEpInSub(void **buf, const SMqConsumerEpInSub *pEpInSub) {
+int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) {
   int32_t tlen = 0;
-  tlen += taosEncodeFixedI64(buf, pEpInSub->consumerId);
-  int32_t sz = taosArrayGetSize(pEpInSub->vgs);
+  tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId);
+  tlen += taosEncodeArray(buf, pConsumerEp->vgs, (FEncode)tEncodeSMqVgEp);
+#if 0
+  int32_t sz = taosArrayGetSize(pConsumerEp->vgs);
   tlen += taosEncodeFixedI32(buf, sz);
   for (int32_t i = 0; i < sz; i++) {
-    SMqVgEp *pVgEp = taosArrayGetP(pEpInSub->vgs, i);
+    SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i);
     tlen += tEncodeSMqVgEp(buf, pVgEp);
   }
-  /*tlen += taosEncodeArray(buf, pEpInSub->vgs, (FEncode)tEncodeSMqVgEp);*/
+#endif
   return tlen;
 }
 
-void *tDecodeSMqConsumerEpInSub(const void *buf, SMqConsumerEpInSub *pEpInSub) {
-  buf = taosDecodeFixedI64(buf, &pEpInSub->consumerId);
-  /*buf = taosDecodeArray(buf, &pEpInSub->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqSubVgEp));*/
+void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp) {
+  buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId);
+  buf = taosDecodeArray(buf, &pConsumerEp->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqSubVgEp));
+#if 0
   int32_t sz;
   buf = taosDecodeFixedI32(buf, &sz);
-  pEpInSub->vgs = taosArrayInit(sz, sizeof(void *));
+  pConsumerEp->vgs = taosArrayInit(sz, sizeof(void *));
   for (int32_t i = 0; i < sz; i++) {
     SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp));
     buf = tDecodeSMqVgEp(buf, pVgEp);
-    taosArrayPush(pEpInSub->vgs, &pVgEp);
+    taosArrayPush(pConsumerEp->vgs, &pVgEp);
   }
+#endif
 
   return (void *)buf;
 }
@@ -217,13 +263,11 @@ SMqSubscribeObj *tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]) {
   taosInitRWLatch(&pSubNew->lock);
   pSubNew->vgNum = 0;
   pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
-  // TODO set free fp
-  SMqConsumerEpInSub epInSub = {
-      .consumerId = -1,
-      .vgs = taosArrayInit(0, sizeof(void *)),
-  };
-  int64_t unexistKey = -1;
-  taosHashPut(pSubNew->consumerHash, &unexistKey, sizeof(int64_t), &epInSub, sizeof(SMqConsumerEpInSub));
+  // TODO set hash free fp
+  /*taosHashSetFreeFp(pSubNew->consumerHash, tDeleteSMqConsumerEp);*/
+
+  pSubNew->unassignedVgs = taosArrayInit(0, sizeof(void *));
+
   return pSubNew;
 }
 
@@ -240,25 +284,27 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) {
 
   pSubNew->vgNum = pSub->vgNum;
   pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
-  /*taosHashSetFreeFp(pSubNew->consumerHash, taosArrayDestroy);*/
+  // TODO set hash free fp
+  /*taosHashSetFreeFp(pSubNew->consumerHash, tDeleteSMqConsumerEp);*/
   void          *pIter = NULL;
-  SMqConsumerEpInSub *pEpInSub = NULL;
+  SMqConsumerEp *pConsumerEp = NULL;
   while (1) {
     pIter = taosHashIterate(pSub->consumerHash, pIter);
     if (pIter == NULL) break;
-    pEpInSub = (SMqConsumerEpInSub *)pIter;
-    SMqConsumerEpInSub newEp = {
-        .consumerId = pEpInSub->consumerId,
-        .vgs = taosArrayDeepCopy(pEpInSub->vgs, (FCopy)tCloneSMqVgEp),
+    pConsumerEp = (SMqConsumerEp *)pIter;
+    SMqConsumerEp newEp = {
+        .consumerId = pConsumerEp->consumerId,
+        .vgs = taosArrayDeepCopy(pConsumerEp->vgs, (FCopy)tCloneSMqVgEp),
     };
-    taosHashPut(pSubNew->consumerHash, &newEp.consumerId, sizeof(int64_t), &newEp, sizeof(SMqConsumerEpInSub));
+    taosHashPut(pSubNew->consumerHash, &newEp.consumerId, sizeof(int64_t), &newEp, sizeof(SMqConsumerEp));
   }
+  pSubNew->unassignedVgs = taosArrayDeepCopy(pSub->unassignedVgs, (FCopy)tCloneSMqVgEp);
   return pSubNew;
 }
 
 void tDeleteSubscribeObj(SMqSubscribeObj *pSub) {
-  /*taosArrayDestroyEx(pSub->consumerEps, (FDelete)tDeleteSMqConsumerEpInSub);*/
   taosHashCleanup(pSub->consumerHash);
+  taosArrayDestroyP(pSub->unassignedVgs, (FDelete)tDeleteSMqVgEp);
 }
 
 int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) {
@@ -278,12 +324,12 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) {
   while (1) {
     pIter = taosHashIterate(pSub->consumerHash, pIter);
     if (pIter == NULL) break;
-    SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter;
-    tlen += tEncodeSMqConsumerEpInSub(buf, pEpInSub);
+    SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
+    tlen += tEncodeSMqConsumerEp(buf, pConsumerEp);
     cnt++;
   }
   ASSERT(cnt == sz);
-  /*tlen += taosEncodeArray(buf, pSub->consumerEps, (FEncode)tEncodeSMqConsumerEpInSub);*/
+  tlen += taosEncodeArray(buf, pSub->unassignedVgs, (FEncode)tEncodeSMqVgEp);
   return tlen;
 }
 
@@ -301,13 +347,12 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) {
 
   pSub->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
   for (int32_t i = 0; i < sz; i++) {
-    /*SMqConsumerEpInSub* pEpInSub = taosMemoryMalloc(sizeof(SMqConsumerEpInSub));*/
-    SMqConsumerEpInSub epInSub = {0};
-    buf = tDecodeSMqConsumerEpInSub(buf, &epInSub);
-    taosHashPut(pSub->consumerHash, &epInSub.consumerId, sizeof(int64_t), &epInSub, sizeof(SMqConsumerEpInSub));
+    SMqConsumerEp consumerEp = {0};
+    buf = tDecodeSMqConsumerEp(buf, &consumerEp);
+    taosHashPut(pSub->consumerHash, &consumerEp.consumerId, sizeof(int64_t), &consumerEp, sizeof(SMqConsumerEp));
   }
 
-  /*buf = taosDecodeArray(buf, &pSub->consumerEps, (FDecode)tDecodeSMqConsumerEpInSub, sizeof(SMqConsumerEpInSub));*/
+  buf = taosDecodeArray(buf, &pSub->unassignedVgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp));
   return (void *)buf;
 }
 
@@ -315,12 +360,12 @@ SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) {
   SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry));
   if (pEntryNew == NULL) return NULL;
   pEntryNew->epoch = pEntry->epoch;
-  pEntryNew->consumers = taosArrayDeepCopy(pEntry->consumers, (FCopy)tCloneSMqConsumerEpInSub);
+  pEntryNew->consumers = taosArrayDeepCopy(pEntry->consumers, (FCopy)tCloneSMqConsumerEp);
   return pEntryNew;
 }
 
 void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) {
-  taosArrayDestroyEx(pEntry->consumers, (FDelete)tDeleteSMqConsumerEpInSub);
+  taosArrayDestroyEx(pEntry->consumers, (FDelete)tDeleteSMqConsumerEp);
 }
 
 int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEntry) {
@@ -329,6 +374,7 @@ int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEnt
   tlen += taosEncodeArray(buf, pEntry->consumers, (FEncode)tEncodeSMqSubActionLogEntry);
   return tlen;
 }
+
 void *tDecodeSMqSubActionLogEntry(const void *buf, SMqSubActionLogEntry *pEntry) {
   buf = taosDecodeFixedI32(buf, &pEntry->epoch);
   buf = taosDecodeArray(buf, &pEntry->consumers, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry));
@@ -339,12 +385,12 @@ SMqSubActionLogObj *tCloneSMqSubActionLogObj(SMqSubActionLogObj *pLog) {
   SMqSubActionLogObj *pLogNew = taosMemoryMalloc(sizeof(SMqSubActionLogObj));
   if (pLogNew == NULL) return pLogNew;
   memcpy(pLogNew->key, pLog->key, TSDB_SUBSCRIBE_KEY_LEN);
-  pLogNew->logs = taosArrayDeepCopy(pLog->logs, (FCopy)tCloneSMqConsumerEpInSub);
+  pLogNew->logs = taosArrayDeepCopy(pLog->logs, (FCopy)tCloneSMqConsumerEp);
   return pLogNew;
 }
 
 void tDeleteSMqSubActionLogObj(SMqSubActionLogObj *pLog) {
-  taosArrayDestroyEx(pLog->logs, (FDelete)tDeleteSMqConsumerEpInSub);
+  taosArrayDestroyEx(pLog->logs, (FDelete)tDeleteSMqConsumerEp);
 }
 
 int32_t tEncodeSMqSubActionLogObj(void **buf, const SMqSubActionLogObj *pLog) {
@@ -364,7 +410,7 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) {
   int32_t sz = 0;
   /*int32_t outputNameSz = 0;*/
   if (tEncodeCStr(pEncoder, pObj->name) < 0) return -1;
-  if (tEncodeCStr(pEncoder, pObj->db) < 0) return -1;
+  if (tEncodeCStr(pEncoder, pObj->sourceDb) < 0) return -1;
   if (tEncodeI64(pEncoder, pObj->createTime) < 0) return -1;
   if (tEncodeI64(pEncoder, pObj->updateTime) < 0) return -1;
   if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1;
@@ -372,6 +418,9 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) {
   if (tEncodeI32(pEncoder, pObj->version) < 0) return -1;
   if (tEncodeI8(pEncoder, pObj->status) < 0) return -1;
   if (tEncodeI8(pEncoder, pObj->createdBy) < 0) return -1;
+  if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1;
+  if (tEncodeI32(pEncoder, pObj->triggerParam) < 0) return -1;
+  if (tEncodeI64(pEncoder, pObj->waterMark) < 0) return -1;
   if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1;
   if (tEncodeI64(pEncoder, pObj->smaId) < 0) return -1;
   if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1;
@@ -410,7 +459,7 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) {
 
 int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) {
   if (tDecodeCStrTo(pDecoder, pObj->name) < 0) return -1;
-  if (tDecodeCStrTo(pDecoder, pObj->db) < 0) return -1;
+  if (tDecodeCStrTo(pDecoder, pObj->sourceDb) < 0) return -1;
   if (tDecodeI64(pDecoder, &pObj->createTime) < 0) return -1;
   if (tDecodeI64(pDecoder, &pObj->updateTime) < 0) return -1;
   if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1;
@@ -418,6 +467,9 @@ int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) {
   if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1;
   if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1;
   if (tDecodeI8(pDecoder, &pObj->createdBy) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pObj->triggerParam) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pObj->waterMark) < 0) return -1;
   if (tDecodeI32(pDecoder, &pObj->fixedSinkVgId) < 0) return -1;
   if (tDecodeI64(pDecoder, &pObj->smaId) < 0) return -1;
   if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1;

source/dnode/mnode/impl/src/mndInfoSchema.c

@@ -14,284 +14,10 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "mndInfoSchema.h"
+#include "systable.h"
 #include "mndInt.h"
 
-#define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
-#define SYSTABLE_SCH_DB_NAME_LEN    ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
-#define SYSTABLE_SCH_COL_NAME_LEN   ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
-
-static const SInfosTableSchema dnodesSchema[] = {
-    {.name = "id", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
-    {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
-    {.name = "max_vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
-    {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "note", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-};
-
-static const SInfosTableSchema mnodesSchema[] = {
-    {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "role", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-};
-
-static const SInfosTableSchema modulesSchema[] = {
-    {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "endpoint", .bytes = 134 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "module", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-};
-
-static const SInfosTableSchema qnodesSchema[] = {
-    {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-};
-
-static const SInfosTableSchema snodesSchema[] = {
-    {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-};
-
-static const SInfosTableSchema bnodesSchema[] = {
-    {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-};
-
-static const SInfosTableSchema clusterSchema[] = {
-    {.name = "id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
-    {.name = "name", .bytes = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-};
-
-static const SInfosTableSchema userDBSchema[] = {
-    {.name = "name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "vgroups", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
-    {.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
-    {.name = "replica", .bytes = 2, .type = TSDB_DATA_TYPE_TINYINT},
-    {.name = "strict", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "days", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "keep", .bytes = 24 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "cache", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "minrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "maxrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "wal", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
-    {.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
-    {.name = "cachelast", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
-    {.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "single_stable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
-    {.name = "stream_mode", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
-    {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    // {.name = "update", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},  // disable update
-};
-
-static const SInfosTableSchema userFuncSchema[] = {
-    {.name = "name", .bytes = TSDB_FUNC_NAME_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "comment", .bytes = PATH_MAX - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "aggregate", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "output_type", .bytes = TSDB_TYPE_STR_MAX_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "code_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "bufsize", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-};
-
-static const SInfosTableSchema userIdxSchema[] = {
-    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "index_database", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "index_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "column_name", .bytes = SYSTABLE_SCH_COL_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_VARCHAR},
-};
-
-static const SInfosTableSchema userStbsSchema[] = {
-    {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "last_update", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "table_comment", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-};
-
-static const SInfosTableSchema userStreamsSchema[] = {
-    {.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "dest_table", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "sql", .bytes = 1024, .type = TSDB_DATA_TYPE_VARCHAR},
-};
-
-static const SInfosTableSchema userTblsSchema[] = {
-    {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "uid", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
-    {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "table_comment", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-};
-
-static const SInfosTableSchema userTblDistSchema[] = {
-    {.name = "db_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "distributed_histogram", .bytes = 500 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "min_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "max_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "avg_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "stddev_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "rows", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
-    {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "storage_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
-    {.name = "compression_ratio", .bytes = 8, .type = TSDB_DATA_TYPE_DOUBLE},
-    {.name = "rows_in_mem", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "seek_header_time", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-};
-
-static const SInfosTableSchema userUsersSchema[] = {
-    {.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "account", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-};
-
-static const SInfosTableSchema grantsSchema[] = {
-    {.name = "version", .bytes = 8 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "expire time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "storage(GB)", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "databases", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "users", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "accounts", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "connections", .bytes = 11 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "streams", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "cpu cores", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "speed(PPS)", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "querytime", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-};
-
-static const SInfosTableSchema vgroupsSchema[] = {
-    {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "v1_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "v1_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "v2_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "v2_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "v3_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "nfiles", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-};
-
-static const SInfosTableSchema consumerSchema[] = {
-    {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "status", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    // ep
-    // up time
-    // topics
-};
-
-static const SInfosTableSchema subscribeSchema[] = {
-    {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-};
-
-static const SInfosTableSchema smaSchema[] = {
-    {.name = "sma_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-};
-
-static const SInfosTableSchema transSchema[] = {
-    {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "stage", .bytes = TSDB_TRANS_STAGE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "type", .bytes = TSDB_TRANS_TYPE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "last_exec_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "last_error", .bytes = (TSDB_TRANS_ERROR_LEN - 1) + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-};
-
-static const SInfosTableSchema configSchema[] = {
-    {.name = "name", .bytes = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "value", .bytes = TSDB_CONIIG_VALUE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-};
-
-static const SInfosTableSchema connSchema[] = {
-    {.name = "connId", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "program", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "ip:port", .bytes = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "login_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "last_access", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-};
-
-static const SInfosTableSchema querySchema[] = {
-    {.name = "queryId", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "connId", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "ip:port", .bytes = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "qid", .bytes = 22 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "time", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
-    {.name = "sql_obj_id", .bytes = QUERY_OBJ_ID_SIZE + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "ep", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "stable_query", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL},
-    {.name = "sub_queries", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "sub_query_info", .bytes = TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-    {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
-};
-
-static const SInfosTableMeta infosMeta[] = {
-    {TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema)},
-    {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema)},
-    {TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema)},
-    {TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema)},
-    {TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema)},
-    {TSDB_INS_TABLE_BNODES, bnodesSchema, tListLen(bnodesSchema)},
-    {TSDB_INS_TABLE_CLUSTER, clusterSchema, tListLen(clusterSchema)},
-    {TSDB_INS_TABLE_USER_DATABASES, userDBSchema, tListLen(userDBSchema)},
-    {TSDB_INS_TABLE_USER_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)},
-    {TSDB_INS_TABLE_USER_INDEXES, userIdxSchema, tListLen(userIdxSchema)},
-    {TSDB_INS_TABLE_USER_STABLES, userStbsSchema, tListLen(userStbsSchema)},
-    {TSDB_INS_TABLE_USER_STREAMS, userStreamsSchema, tListLen(userStreamsSchema)},
-    {TSDB_INS_TABLE_USER_TABLES, userTblsSchema, tListLen(userTblsSchema)},
-    {TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)},
-    {TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)},
-    {TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema)},
-    {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)},
-    {TSDB_INS_TABLE_CONSUMERS, consumerSchema, tListLen(consumerSchema)},
-    {TSDB_INS_TABLE_SUBSCRIBES, subscribeSchema, tListLen(subscribeSchema)},
-    {TSDB_INS_TABLE_TRANS, transSchema, tListLen(transSchema)},
-    {TSDB_INS_TABLE_SMAS, smaSchema, tListLen(smaSchema)},
-    {TSDB_INS_TABLE_CONFIGS, configSchema, tListLen(configSchema)},
-    {TSDB_INS_TABLE_CONNS, connSchema, tListLen(connSchema)},
-    {TSDB_INS_TABLE_QUERIES, querySchema, tListLen(querySchema)},
-};
-
-static int32_t mndInitInfosTableSchema(const SInfosTableSchema *pSrc, int32_t colNum, SSchema **pDst) {
+static int32_t mndInitInfosTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, SSchema **pDst) {
   SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema));
   if (NULL == schema) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -317,11 +43,15 @@ static int32_t mndInsInitMeta(SHashObj *hash) {
   meta.sversion = 1;
   meta.tversion = 1;
 
-  for (int32_t i = 0; i < tListLen(infosMeta); ++i) {
-    tstrncpy(meta.tbName, infosMeta[i].name, sizeof(meta.tbName));
-    meta.numOfColumns = infosMeta[i].colNum;
+  size_t size = 0;
+  const SSysTableMeta* pInfosTableMeta = NULL;
+  getInfosDbMeta(&pInfosTableMeta, &size);
+
+  for (int32_t i = 0; i < size; ++i) {
+    tstrncpy(meta.tbName, pInfosTableMeta[i].name, sizeof(meta.tbName));
+    meta.numOfColumns = pInfosTableMeta[i].colNum;
 
-    if (mndInitInfosTableSchema(infosMeta[i].schema, infosMeta[i].colNum, &meta.pSchemas)) {
+    if (mndInitInfosTableSchema(pInfosTableMeta[i].schema, pInfosTableMeta[i].colNum, &meta.pSchemas)) {
       return -1;
     }
 

source/dnode/mnode/impl/src/mndPerfSchema.c

@@ -14,70 +14,11 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "mndPerfSchema.h"
 #include "mndInt.h"
-
-//!!!! Note: only APPEND columns in below tables, NO insert !!!!
-static const SPerfsTableSchema connectionsSchema[] = {
-    {.name = "conn_id", .bytes = 4, .type = TSDB_DATA_TYPE_UINT},
-    {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
-    {.name = "program", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
-    {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "end_point", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
-    {.name = "login_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "last_access", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-};
-static const SPerfsTableSchema queriesSchema[] = {
-    {.name = "query_id", .bytes = 4, .type = TSDB_DATA_TYPE_UBIGINT},
-    {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
-    {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
-    {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "fqdn", .bytes = TSDB_FQDN_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
-    {.name = "exec_time", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
-    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "sub_queries", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "sub_query_info", .bytes = TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
-};
-
-static const SPerfsTableSchema topicSchema[] = {
-    {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
-    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
-    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-    {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
-    // TODO config
-};
-
-static const SPerfsTableSchema consumerSchema[] = {
-    {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
-    {.name = "app_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
-    {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
-    {.name = "status", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "topics", .bytes = TSDB_SHOW_LIST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
-    {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "end_point", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
-    {.name = "up_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
-};
-
-static const SPerfsTableSchema subscriptionSchema[] = {
-    {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
-    {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
-    {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
-    {.name = "committed_offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
-    {.name = "current_offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
-    {.name = "skip_log_cnt", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
-};
-
-static const SPerfsTableMeta perfsMeta[] = {
-    {TSDB_PERFS_TABLE_CONNECTIONS, connectionsSchema, tListLen(connectionsSchema)},
-    {TSDB_PERFS_TABLE_QUERIES, queriesSchema, tListLen(queriesSchema)},
-    {TSDB_PERFS_TABLE_TOPICS, topicSchema, tListLen(topicSchema)},
-    {TSDB_PERFS_TABLE_CONSUMERS, consumerSchema, tListLen(consumerSchema)},
-    {TSDB_PERFS_TABLE_SUBSCRIPTIONS, subscriptionSchema, tListLen(subscriptionSchema)},
-};
+#include "systable.h"
 
 // connection/application/
-int32_t mndInitPerfsTableSchema(const SPerfsTableSchema *pSrc, int32_t colNum, SSchema **pDst) {
+int32_t mndInitPerfsTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, SSchema **pDst) {
   SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema));
   if (NULL == schema) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -104,11 +45,15 @@ int32_t mndPerfsInitMeta(SHashObj *hash) {
   meta.sversion = 1;
   meta.tversion = 1;
 
-  for (int32_t i = 0; i < tListLen(perfsMeta); ++i) {
-    strcpy(meta.tbName, perfsMeta[i].name);
-    meta.numOfColumns = perfsMeta[i].colNum;
+  size_t size = 0;
+  const SSysTableMeta* pSysDbTableMeta = NULL;
+  getPerfDbMeta(&pSysDbTableMeta, &size);
+
+  for (int32_t i = 0; i < size; ++i) {
+    strcpy(meta.tbName, pSysDbTableMeta[i].name);
+    meta.numOfColumns = pSysDbTableMeta[i].colNum;
 
-    if (mndInitPerfsTableSchema(perfsMeta[i].schema, perfsMeta[i].colNum, &meta.pSchemas)) {
+    if (mndInitPerfsTableSchema(pSysDbTableMeta[i].schema, pSysDbTableMeta[i].colNum, &meta.pSchemas)) {
       return -1;
     }
 
@@ -144,7 +89,6 @@ int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char
   }
 
   memcpy(pRsp->pSchemas, meta->pSchemas, meta->numOfColumns * sizeof(SSchema));
-
   return 0;
 }
 

source/dnode/mnode/impl/src/mndProfile.c

@@ -196,7 +196,7 @@ static int32_t mndProcessConnectReq(SNodeMsg *pReq) {
     goto CONN_OVER;
   }
   if (0 != strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1)) {
-    mError("user:%s, failed to auth while acquire user\n %s \r\n %s", pReq->user, connReq.passwd, pUser->pass);
+    mError("user:%s, failed to auth while acquire user, input:%s saved:%s", pReq->user, connReq.passwd, pUser->pass);
     code = TSDB_CODE_RPC_AUTH_FAILURE;
     goto CONN_OVER;
   }

source/dnode/mnode/impl/src/mndQuery.c

@@ -20,7 +20,7 @@
 
 int32_t mndProcessQueryMsg(SNodeMsg *pReq) {
   SMnode     *pMnode = pReq->pNode;
-  SReadHandle handle = {0};
+  SReadHandle handle = {.mnd = pMnode};
 
   mTrace("msg:%p, in query queue is processing", pReq);
   switch (pReq->rpcMsg.msgType) {

source/dnode/mnode/impl/src/mndScheduler.c

@@ -308,8 +308,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
         // sink part
         if (level == 0) {
           // only for inplace
-          pTask->sinkType = TASK_SINK__SHOW;
-          pTask->showSink.reserved = 0;
+          pTask->sinkType = TASK_SINK__NONE;
           if (!hasExtraSink) {
 #if 1
             if (pStream->createdBy == STREAM_CREATED_BY__SMA) {
@@ -368,8 +367,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
       pTask->sourceType = TASK_SOURCE__PIPE;
 
       // sink part
-      pTask->sinkType = TASK_SINK__SHOW;
-      /*pTask->sinkType = TASK_SINK__NONE;*/
+      pTask->sinkType = TASK_SINK__NONE;
 
       // dispatch part
       ASSERT(hasExtraSink);
@@ -382,7 +380,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
           pTask->dispatchType = TASK_DISPATCH__SHUFFLE;
 
           pTask->dispatchMsgType = TDMT_VND_TASK_WRITE_EXEC;
-          SDbObj* pDb = mndAcquireDb(pMnode, pStream->db);
+          SDbObj* pDb = mndAcquireDb(pMnode, pStream->sourceDb);
           ASSERT(pDb);
           if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) {
             sdbRelease(pSdb, pDb);
@@ -456,7 +454,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
       pTask->sourceType = TASK_SOURCE__MERGE;
 
       // sink part
-      pTask->sinkType = TASK_SINK__SHOW;
+      pTask->sinkType = TASK_SINK__NONE;
 
       // dispatch part
       pTask->dispatchType = TASK_DISPATCH__NONE;
@@ -489,7 +487,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     int32_t levelNum = LIST_LENGTH(pPlan->pSubplans);
     if (levelNum != 1) {
       qDestroyQueryPlan(pPlan);
-      terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;
+      terrno = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
       return -1;
     }
 
@@ -498,17 +496,14 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     int32_t opNum = LIST_LENGTH(inner->pNodeList);
     if (opNum != 1) {
       qDestroyQueryPlan(pPlan);
-      terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;
+      terrno = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
       return -1;
     }
     plan = nodesListGetNode(inner->pNodeList, 0);
   }
 
-  int64_t             unexistKey = -1;
-  SMqConsumerEpInSub* pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t));
-  ASSERT(pEpInSub);
-
-  ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
+  ASSERT(pSub->unassignedVgs);
+  ASSERT(taosHashGetSize(pSub->consumerHash) == 0);
 
   void* pIter = NULL;
   while (1) {
@@ -524,7 +519,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     SMqVgEp* pVgEp = taosMemoryMalloc(sizeof(SMqVgEp));
     pVgEp->epSet = mndGetVgroupEpset(pMnode, pVgroup);
     pVgEp->vgId = pVgroup->vgId;
-    taosArrayPush(pEpInSub->vgs, &pVgEp);
+    taosArrayPush(pSub->unassignedVgs, &pVgEp);
 
     mDebug("init subscription %s, assign vg: %d", pSub->key, pVgEp->vgId);
 
@@ -543,17 +538,11 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
     } else {
       pVgEp->qmsg = strdup("");
     }
-
-    ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
-
-    /*taosArrayPush(pSub->unassignedVg, &consumerEp);*/
   }
 
-  pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t));
-
-  ASSERT(pEpInSub->vgs->size > 0);
+  ASSERT(pSub->unassignedVgs->size > 0);
 
-  ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
+  ASSERT(taosHashGetSize(pSub->consumerHash) == 0);
 
   qDestroyQueryPlan(pPlan);
 

source/dnode/mnode/impl/src/mndShow.c

@@ -15,6 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "mndShow.h"
+#include "systable.h"
 
 #define SHOW_STEP_SIZE 100
 
@@ -47,7 +48,7 @@ void mndCleanupShow(SMnode *pMnode) {
   }
 }
 
-static int32_t convertToRetrieveType(char* name, int32_t len) {
+static int32_t convertToRetrieveType(char *name, int32_t len) {
   int32_t type = -1;
 
   if (strncasecmp(name, TSDB_INS_TABLE_DNODES, len) == 0) {
@@ -72,8 +73,6 @@ static int32_t convertToRetrieveType(char* name, int32_t len) {
     //    type = TSDB_MGMT_TABLE_INDEX;
   } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, len) == 0) {
     type = TSDB_MGMT_TABLE_STB;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STREAMS, len) == 0) {
-    type = TSDB_MGMT_TABLE_STREAMS;
   } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, len) == 0) {
     type = TSDB_MGMT_TABLE_TABLE;
   } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, len) == 0) {
@@ -84,26 +83,28 @@ static int32_t convertToRetrieveType(char* name, int32_t len) {
     type = TSDB_MGMT_TABLE_GRANTS;
   } else if (strncasecmp(name, TSDB_INS_TABLE_VGROUPS, len) == 0) {
     type = TSDB_MGMT_TABLE_VGROUP;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_CONSUMERS, len) == 0) {
+  } else if (strncasecmp(name, TSDB_PERFS_TABLE_CONSUMERS, len) == 0) {
     type = TSDB_MGMT_TABLE_CONSUMERS;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_SUBSCRIBES, len) == 0) {
+  } else if (strncasecmp(name, TSDB_PERFS_TABLE_SUBSCRIBES, len) == 0) {
     type = TSDB_MGMT_TABLE_SUBSCRIBES;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_TRANS, len) == 0) {
+  } else if (strncasecmp(name, TSDB_PERFS_TABLE_TRANS, len) == 0) {
     type = TSDB_MGMT_TABLE_TRANS;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_SMAS, len) == 0) {
+  } else if (strncasecmp(name, TSDB_PERFS_TABLE_SMAS, len) == 0) {
     type = TSDB_MGMT_TABLE_SMAS;
   } else if (strncasecmp(name, TSDB_INS_TABLE_CONFIGS, len) == 0) {
     type = TSDB_MGMT_TABLE_CONFIGS;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_CONNS, len) == 0) {
+  } else if (strncasecmp(name, TSDB_PERFS_TABLE_CONNECTIONS, len) == 0) {
     type = TSDB_MGMT_TABLE_CONNS;
-  } else if (strncasecmp(name, TSDB_INS_TABLE_QUERIES, len) == 0) {
+  } else if (strncasecmp(name, TSDB_PERFS_TABLE_QUERIES, len) == 0) {
     type = TSDB_MGMT_TABLE_QUERIES;
   } else if (strncasecmp(name, TSDB_INS_TABLE_VNODES, len) == 0) {
     type = TSDB_MGMT_TABLE_VNODES;
   } else if (strncasecmp(name, TSDB_PERFS_TABLE_TOPICS, len) == 0) {
     type = TSDB_MGMT_TABLE_TOPICS;
+  } else if (strncasecmp(name, TSDB_PERFS_TABLE_STREAMS, len) == 0) {
+    type = TSDB_MGMT_TABLE_STREAMS;
   } else {
-//    ASSERT(0);
+    //    ASSERT(0);
   }
 
   return type;

source/dnode/mnode/impl/src/mndSma.c

@@ -40,7 +40,7 @@ static int32_t  mndProcessMCreateSmaReq(SNodeMsg *pReq);
 static int32_t  mndProcessMDropSmaReq(SNodeMsg *pReq);
 static int32_t  mndProcessVCreateSmaRsp(SNodeMsg *pRsp);
 static int32_t  mndProcessVDropSmaRsp(SNodeMsg *pRsp);
-static int32_t  mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows);
+static int32_t  mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void     mndCancelGetNextSma(SMnode *pMnode, void *pIter);
 
 int32_t mndInitSma(SMnode *pMnode) {
@@ -406,7 +406,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCre
 
   SStreamObj streamObj = {0};
   tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN);
-  tstrncpy(streamObj.db, pDb->name, TSDB_DB_FNAME_LEN);
+  tstrncpy(streamObj.sourceDb, pDb->name, TSDB_DB_FNAME_LEN);
   streamObj.createTime = taosGetTimestampMs();
   streamObj.updateTime = streamObj.createTime;
   streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name));
@@ -707,7 +707,8 @@ int32_t mndProcessGetSmaReq(SMnode      *pMnode, SUserIndexReq *indexReq, SUserI
     SNode *node = NULL;
     FOREACH(node, pList) {
       SFunctionNode *pFunc = (SFunctionNode *)node;
-      extOffset += snprintf(rsp->indexExts + extOffset, sizeof(rsp->indexExts) - extOffset - 1, "%s%s", (extOffset ? ",":""), pFunc->functionName);
+      extOffset += snprintf(rsp->indexExts + extOffset, sizeof(rsp->indexExts) - extOffset - 1, "%s%s",
+                            (extOffset ? "," : ""), pFunc->functionName);
     }
 
     *exist = true;
@@ -718,13 +719,12 @@ int32_t mndProcessGetSmaReq(SMnode      *pMnode, SUserIndexReq *indexReq, SUserI
   return code;
 }
 
-
 static int32_t mndProcessVDropSmaRsp(SNodeMsg *pRsp) {
   mndTransProcessRsp(pRsp);
   return 0;
 }
 
-static int32_t mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) {
+static int32_t mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
   SMnode  *pMnode = pReq->pNode;
   SSdb    *pSdb = pMnode->pSdb;
   int32_t  numOfRows = 0;
@@ -758,8 +758,8 @@ static int32_t mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlo
     char n1[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
     STR_TO_VARSTR(n1, (char *)tNameGetTableName(&stbName));
 
-    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char*) n, false);
+    SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)n, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, (const char *)&pSma->createdTime, false);

source/dnode/mnode/impl/src/mndStb.c

@@ -395,7 +395,9 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
     }
   }
   // get length
-  if (tEncodeSize(tEncodeSVCreateStbReq, &req, contLen) < 0) {
+  int32_t ret = 0;
+  tEncodeSize(tEncodeSVCreateStbReq, &req, contLen, ret);
+  if (ret < 0) {
     return NULL;
   }
 
@@ -772,6 +774,13 @@ static int32_t mndProcessMCreateStbReq(SNodeMsg *pReq) {
     goto _OVER;
   }
 
+  int32_t numOfStbs = -1;
+  mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs);
+  if (pDb->cfg.numOfStables == 1 && numOfStbs != 0 ) {
+    terrno = TSDB_CODE_MND_SINGLE_STB_MODE_DB;
+    goto _OVER;
+  }
+
   code = mndCreateStb(pMnode, pReq, &createReq, pDb);
   if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
 
@@ -1425,7 +1434,6 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa
   pRsp->numOfColumns = pStb->numOfColumns;
   pRsp->precision = pDb->cfg.precision;
   pRsp->tableType = TSDB_SUPER_TABLE;
-  pRsp->update = pDb->cfg.update;
   pRsp->sversion = pStb->version;
   pRsp->suid = pStb->uid;
   pRsp->tuid = pStb->uid;
@@ -1578,7 +1586,7 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbVersions, int
   return 0;
 }
 
-static int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs) {
+int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs) {
   SSdb   *pSdb = pMnode->pSdb;
   SDbObj *pDb = mndAcquireDb(pMnode, dbName);
   if (pDb == NULL) {

source/dnode/mnode/impl/src/mndStream.c

@@ -40,7 +40,7 @@ static int32_t mndProcessTaskDeployInternalRsp(SNodeMsg *pRsp);
 /*static int32_t mndProcessDropStreamInRsp(SNodeMsg *pRsp);*/
 static int32_t mndProcessStreamMetaReq(SNodeMsg *pReq);
 static int32_t mndGetStreamMeta(SNodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta);
-static int32_t mndRetrieveStream(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows);
+static int32_t mndRetrieveStream(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextStream(SMnode *pMnode, void *pIter);
 
 int32_t mndInitStream(SMnode *pMnode) {
@@ -58,8 +58,8 @@ int32_t mndInitStream(SMnode *pMnode) {
   /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq);*/
   /*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM_RSP, mndProcessDropStreamInRsp);*/
 
-  //  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndRetrieveStream);
-  /*mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextStream);*/
+  mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndRetrieveStream);
+  mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndCancelGetNextStream);
 
   return sdbSetTable(pMnode->pSdb, table);
 }
@@ -294,8 +294,8 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe
   mDebug("stream:%s to create", pCreate->name);
   SStreamObj streamObj = {0};
   tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN);
-  tstrncpy(streamObj.db, pDb->name, TSDB_DB_FNAME_LEN);
-  tstrncpy(streamObj.outputSTbName, pCreate->outputSTbName, TSDB_TABLE_FNAME_LEN);
+  tstrncpy(streamObj.sourceDb, pDb->name, TSDB_DB_FNAME_LEN);
+  tstrncpy(streamObj.targetSTbName, pCreate->targetStbFullName, TSDB_TABLE_FNAME_LEN);
   streamObj.createTime = taosGetTimestampMs();
   streamObj.updateTime = streamObj.createTime;
   streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name));
@@ -308,6 +308,8 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe
   streamObj.smaId = 0;
   /*streamObj.physicalPlan = "";*/
   streamObj.logicalPlan = "not implemented";
+  streamObj.trigger = pCreate->triggerType;
+  streamObj.waterMark = pCreate->watermark;
 
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_STREAM, &pReq->rpcMsg);
   if (pTrans == NULL) {
@@ -424,56 +426,58 @@ static int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfS
   return 0;
 }
 
-static int32_t mndRetrieveStream(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) {
+static int32_t mndRetrieveStream(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
   SMnode     *pMnode = pReq->pNode;
   SSdb       *pSdb = pMnode->pSdb;
   int32_t     numOfRows = 0;
   SStreamObj *pStream = NULL;
-  int32_t     cols = 0;
-  char       *pWrite;
-  char        prefix[TSDB_DB_FNAME_LEN] = {0};
-
-  SDbObj *pDb = mndAcquireDb(pMnode, pShow->db);
-  if (pDb == NULL) return 0;
-
-  tstrncpy(prefix, pShow->db, TSDB_DB_FNAME_LEN);
-  strcat(prefix, TS_PATH_DELIMITER);
-  int32_t prefixLen = (int32_t)strlen(prefix);
 
   while (numOfRows < rows) {
     pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream);
     if (pShow->pIter == NULL) break;
 
-    if (pStream->dbUid != pDb->uid) {
-      if (strncmp(pStream->name, prefix, prefixLen) != 0) {
-        mError("Inconsistent stream data, name:%s, db:%s, dbUid:%" PRIu64, pStream->name, pDb->name, pDb->uid);
-      }
+    SColumnInfoData *pColInfo;
+    SName            n;
+    int32_t          cols = 0;
 
-      sdbRelease(pSdb, pStream);
-      continue;
-    }
+    char streamName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
+    tNameFromString(&n, pStream->name, T_NAME_ACCT | T_NAME_DB);
+    tNameGetDbName(&n, varDataVal(streamName));
+    varDataSetLen(streamName, strlen(varDataVal(streamName)));
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)streamName, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pStream->createTime, false);
+
+    char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
+    tstrncpy(&sql[VARSTR_HEADER_SIZE], pStream->sql, TSDB_SHOW_SQL_LEN);
+    varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE]));
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)sql, false);
 
-    cols = 0;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pStream->status, true);
 
-    char streamName[TSDB_TABLE_NAME_LEN] = {0};
-    tstrncpy(streamName, pStream->name + prefixLen, TSDB_TABLE_NAME_LEN);
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    STR_TO_VARSTR(pWrite, streamName);
-    cols++;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pStream->sourceDb, true);
 
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int64_t *)pWrite = pStream->createTime;
-    cols++;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pStream->targetDb, true);
 
-    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pStream->sql, pShow->bytes[cols]);
-    cols++;
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pStream->targetSTbName, true);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pStream->waterMark, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pStream->trigger, false);
 
     numOfRows++;
     sdbRelease(pSdb, pStream);
   }
 
-  mndReleaseDb(pMnode, pDb);
   pShow->numOfRows += numOfRows;
   return numOfRows;
 }

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -85,7 +85,8 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic,
   pSub->withSchema = pTopic->withSchema;
   pSub->withTag = pTopic->withTag;
 
-  ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
+  ASSERT(pSub->unassignedVgs->size == 0);
+  ASSERT(taosHashGetSize(pSub->consumerHash) == 0);
 
   if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) {
     tDeleteSubscribeObj(pSub);
@@ -93,7 +94,8 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic,
     return NULL;
   }
 
-  ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
+  ASSERT(pSub->unassignedVgs->size > 0);
+  ASSERT(taosHashGetSize(pSub->consumerHash) == 0);
 
   return pSub;
 }
@@ -185,7 +187,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
   if (pInput->pTopic != NULL) {
     // create subscribe
     pOutput->pSub = mndCreateSub(pMnode, pInput->pTopic, pInput->pRebInfo->key);
-    ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) == 1);
+    ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) == 0);
   } else {
     pOutput->pSub = tCloneSubscribeObj(pInput->pOldSub);
   }
@@ -196,21 +198,20 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
   // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg
   SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
 
-  ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) > 0);
   // 2. check and get actual removed consumers, put their vg into hash
   int32_t removedNum = taosArrayGetSize(pInput->pRebInfo->removedConsumers);
   int32_t actualRemoved = 0;
   for (int32_t i = 0; i < removedNum; i++) {
     int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i);
     ASSERT(consumerId > 0);
-    SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
-    ASSERT(pEpInSub);
-    if (pEpInSub) {
-      ASSERT(consumerId == pEpInSub->consumerId);
+    SMqConsumerEp *pConsumerEp = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
+    ASSERT(pConsumerEp);
+    if (pConsumerEp) {
+      ASSERT(consumerId == pConsumerEp->consumerId);
       actualRemoved++;
-      int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);
+      int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs);
       for (int32_t j = 0; j < consumerVgNum; j++) {
-        SMqVgEp       *pVgEp = taosArrayGetP(pEpInSub->vgs, j);
+        SMqVgEp       *pVgEp = taosArrayGetP(pConsumerEp->vgs, j);
         SMqRebOutputVg outputVg = {
             .oldConsumerId = consumerId,
             .newConsumerId = -1,
@@ -224,16 +225,12 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
     }
   }
   ASSERT(removedNum == actualRemoved);
-  ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) > 0);
 
   // if previously no consumer, there are vgs not assigned
   {
-    int64_t             unexistKey = -1;
-    SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &unexistKey, sizeof(int64_t));
-    ASSERT(pEpInSub);
-    int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);
+    int32_t consumerVgNum = taosArrayGetSize(pOutput->pSub->unassignedVgs);
     for (int32_t i = 0; i < consumerVgNum; i++) {
-      SMqVgEp       *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs);
+      SMqVgEp       *pVgEp = *(SMqVgEp **)taosArrayPop(pOutput->pSub->unassignedVgs);
       SMqRebOutputVg rebOutput = {
           .oldConsumerId = -1,
           .newConsumerId = -1,
@@ -246,7 +243,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
   // 3. calc vg number of each consumer
   int32_t oldSz = 0;
   if (pInput->pOldSub) {
-    oldSz = taosHashGetSize(pInput->pOldSub->consumerHash) - 1;
+    oldSz = taosHashGetSize(pInput->pOldSub->consumerHash);
   }
   int32_t afterRebConsumerNum =
       oldSz + taosArrayGetSize(pInput->pRebInfo->newConsumers) - taosArrayGetSize(pInput->pRebInfo->removedConsumers);
@@ -264,23 +261,22 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
   while (1) {
     pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
     if (pIter == NULL) break;
-    SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter;
-    if (pEpInSub->consumerId == -1) continue;
-    ASSERT(pEpInSub->consumerId > 0);
-    int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);
+    SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
+    ASSERT(pConsumerEp->consumerId > 0);
+    int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs);
     // all old consumers still existing are touched
     // TODO optimize: touch only consumer whose vgs changed
-    taosArrayPush(pOutput->touchedConsumers, &pEpInSub->consumerId);
+    taosArrayPush(pOutput->touchedConsumers, &pConsumerEp->consumerId);
     if (consumerVgNum > minVgCnt) {
       if (imbCnt < imbConsumerNum) {
         if (consumerVgNum == minVgCnt + 1) {
           continue;
         } else {
           // pop until equal minVg + 1
-          while (taosArrayGetSize(pEpInSub->vgs) > minVgCnt + 1) {
-            SMqVgEp       *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs);
+          while (taosArrayGetSize(pConsumerEp->vgs) > minVgCnt + 1) {
+            SMqVgEp       *pVgEp = *(SMqVgEp **)taosArrayPop(pConsumerEp->vgs);
             SMqRebOutputVg outputVg = {
-                .oldConsumerId = pEpInSub->consumerId,
+                .oldConsumerId = pConsumerEp->consumerId,
                 .newConsumerId = -1,
                 .pVgEp = pVgEp,
             };
@@ -290,10 +286,10 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
         }
       } else {
         // pop until equal minVg
-        while (taosArrayGetSize(pEpInSub->vgs) > minVgCnt) {
-          SMqVgEp       *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs);
+        while (taosArrayGetSize(pConsumerEp->vgs) > minVgCnt) {
+          SMqVgEp       *pVgEp = *(SMqVgEp **)taosArrayPop(pConsumerEp->vgs);
           SMqRebOutputVg outputVg = {
-              .oldConsumerId = pEpInSub->consumerId,
+              .oldConsumerId = pConsumerEp->consumerId,
               .newConsumerId = -1,
               .pVgEp = pVgEp,
           };
@@ -309,14 +305,10 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
     for (int32_t i = 0; i < consumerNum; i++) {
       int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i);
       ASSERT(consumerId > 0);
-      SMqConsumerEpInSub newConsumerEp;
+      SMqConsumerEp newConsumerEp;
       newConsumerEp.consumerId = consumerId;
       newConsumerEp.vgs = taosArrayInit(0, sizeof(void *));
-      taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp,
-                  sizeof(SMqConsumerEpInSub));
-      /*SMqConsumerEpInSub *pTestNew = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));*/
-      /*ASSERT(pTestNew->consumerId == consumerId);*/
-      /*ASSERT(pTestNew->vgs == newConsumerEp.vgs);*/
+      taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp));
       taosArrayPush(pOutput->newConsumers, &consumerId);
     }
   }
@@ -329,25 +321,24 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
   while (1) {
     pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
     if (pIter == NULL) break;
-    SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter;
-    if (pEpInSub->consumerId == -1) continue;
-    ASSERT(pEpInSub->consumerId > 0);
+    SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
+    ASSERT(pConsumerEp->consumerId > 0);
 
     // push until equal minVg
-    while (taosArrayGetSize(pEpInSub->vgs) < minVgCnt) {
+    while (taosArrayGetSize(pConsumerEp->vgs) < minVgCnt) {
       // iter hash and find one vg
       pRemovedIter = taosHashIterate(pHash, pRemovedIter);
       ASSERT(pRemovedIter);
       pRebVg = (SMqRebOutputVg *)pRemovedIter;
       // push
-      taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp);
-      pRebVg->newConsumerId = pEpInSub->consumerId;
+      taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
+      pRebVg->newConsumerId = pConsumerEp->consumerId;
       taosArrayPush(pOutput->rebVgs, pRebVg);
     }
   }
 
   // 7. handle unassigned vg
-  if (taosHashGetSize(pOutput->pSub->consumerHash) != 1) {
+  if (taosHashGetSize(pOutput->pSub->consumerHash) != 0) {
     // if has consumer, assign all left vg
     while (1) {
       pRemovedIter = taosHashIterate(pHash, pRemovedIter);
@@ -355,20 +346,14 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
       pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
       ASSERT(pIter);
       pRebVg = (SMqRebOutputVg *)pRemovedIter;
-      SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter;
-      if (pEpInSub->consumerId == -1) continue;
-      ASSERT(pEpInSub->consumerId > 0);
-      taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp);
-      pRebVg->newConsumerId = pEpInSub->consumerId;
+      SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
+      ASSERT(pConsumerEp->consumerId > 0);
+      taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
+      pRebVg->newConsumerId = pConsumerEp->consumerId;
       taosArrayPush(pOutput->rebVgs, pRebVg);
     }
   } else {
     // if all consumer is removed, put all vg into unassigned
-    int64_t             unexistKey = -1;
-    SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &unexistKey, sizeof(int64_t));
-    ASSERT(pEpInSub);
-    ASSERT(pEpInSub->consumerId == -1);
-
     pIter = NULL;
     SMqRebOutputVg *pRebOutput = NULL;
     while (1) {
@@ -376,12 +361,18 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
       if (pIter == NULL) break;
       pRebOutput = (SMqRebOutputVg *)pIter;
       ASSERT(pRebOutput->newConsumerId == -1);
-      taosArrayPush(pEpInSub->vgs, &pRebOutput->pVgEp);
+      taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp);
       taosArrayPush(pOutput->rebVgs, pRebOutput);
     }
   }
 
-  // 8. generate logs
+  // 8. TODO generate logs
+  mInfo("rebalance calculation completed, rebalanced vg:");
+  for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) {
+    SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i);
+    mInfo("vg: %d moved from consumer %ld to consumer %ld", pOutputRebVg->pVgEp->vgId, pOutputRebVg->oldConsumerId,
+          pOutputRebVg->newConsumerId);
+  }
 
   // 9. clear
   taosHashCleanup(pHash);
@@ -459,7 +450,12 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SNodeMsg *pMsg, const SMqRebO
       goto REB_FAIL;
     }
   }
-  // 4. commit log: modification log
+  // 4. TODO commit log: modification log
+
+  // 5. set cb
+  mndTransSetCb(pTrans, MQ_REB_TRANS_START_FUNC, MQ_REB_TRANS_STOP_FUNC, NULL, 0);
+
+  // 6. execution
   if (mndTransPrepare(pMnode, pTrans) != 0) goto REB_FAIL;
 
   mndTransDrop(pTrans);
@@ -512,6 +508,7 @@ static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) {
     // possibly no vg is changed
     /*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/
 
+    // TODO replace assert with error check
     ASSERT(mndPersistRebResult(pMnode, pMsg, &rebOutput) == 0);
 
     if (rebInput.pTopic) {
@@ -524,9 +521,9 @@ static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) {
   }
 
   // reset flag
-  atomic_store_8(pReq->mqInReb, 0);
   mInfo("mq rebalance completed successfully");
   taosHashCleanup(pReq->rebSubHash);
+  mndRebEnd();
 
   return 0;
 }
@@ -631,6 +628,10 @@ static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubsc
   pOldSub->consumerHash = pNewSub->consumerHash;
   pNewSub->consumerHash = tmp;
 
+  SArray *tmp1 = pOldSub->unassignedVgs;
+  pOldSub->unassignedVgs = pNewSub->unassignedVgs;
+  pNewSub->unassignedVgs = tmp1;
+
   taosWUnLockLatch(&pOldSub->lock);
   return 0;
 }

source/dnode/mnode/impl/src/mndTopic.c

@@ -261,7 +261,7 @@ static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMq
 
 static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) {
   if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0 || pCreate->subscribeDbName[0] == 0) {
-    terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION;
+    terrno = TSDB_CODE_MND_INVALID_TOPIC;
     return -1;
   }
 

source/dnode/mnode/impl/src/mndTrans.c

@@ -16,6 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "mndTrans.h"
 #include "mndAuth.h"
+#include "mndConsumer.h"
 #include "mndDb.h"
 #include "mndShow.h"
 #include "mndSync.h"
@@ -442,6 +443,10 @@ static TransCbFp mndTransGetCbFp(ETrnFuncType ftype) {
       return mndTransTestStartFunc;
     case TEST_TRANS_STOP_FUNC:
       return mndTransTestStopFunc;
+    case MQ_REB_TRANS_START_FUNC:
+      return mndRebCntInc;
+    case MQ_REB_TRANS_STOP_FUNC:
+      return mndRebCntDec;
     default:
       return NULL;
   }

source/dnode/mnode/impl/src/mndVgroup.c

@@ -190,25 +190,24 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg
   memcpy(createReq.db, pDb->name, TSDB_DB_FNAME_LEN);
   createReq.dbUid = pDb->uid;
   createReq.vgVersion = pVgroup->version;
-  createReq.cacheBlockSize = pDb->cfg.cacheBlockSize;
-  createReq.totalBlocks = pDb->cfg.totalBlocks;
+  createReq.numOfStables = pDb->cfg.numOfStables;
+  createReq.buffer = pDb->cfg.buffer;
+  createReq.pageSize = pDb->cfg.pageSize;
+  createReq.pages = pDb->cfg.pages;
   createReq.daysPerFile = pDb->cfg.daysPerFile;
   createReq.daysToKeep0 = pDb->cfg.daysToKeep0;
   createReq.daysToKeep1 = pDb->cfg.daysToKeep1;
   createReq.daysToKeep2 = pDb->cfg.daysToKeep2;
   createReq.minRows = pDb->cfg.minRows;
   createReq.maxRows = pDb->cfg.maxRows;
-  createReq.commitTime = pDb->cfg.commitTime;
   createReq.fsyncPeriod = pDb->cfg.fsyncPeriod;
   createReq.walLevel = pDb->cfg.walLevel;
   createReq.precision = pDb->cfg.precision;
   createReq.compression = pDb->cfg.compression;
   createReq.strict = pDb->cfg.strict;
-  createReq.update = pDb->cfg.update;
   createReq.cacheLastRow = pDb->cfg.cacheLastRow;
   createReq.replica = pVgroup->replica;
   createReq.selfIndex = -1;
-  createReq.streamMode = pVgroup->streamMode;
   createReq.hashBegin = pVgroup->hashBegin;
   createReq.hashEnd = pVgroup->hashEnd;
   createReq.hashMethod = pDb->cfg.hashMethod;
@@ -398,7 +397,6 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
     pVgroup->createdTime = taosGetTimestampMs();
     pVgroup->updateTime = pVgroups->createdTime;
     pVgroup->version = 1;
-    pVgroup->streamMode = pDb->cfg.streamMode;
     pVgroup->hashBegin = hashMin + hashInterval * v;
     if (v == pDb->cfg.numOfVgroups - 1) {
       pVgroup->hashEnd = hashMax;

source/dnode/mnode/impl/test/CMakeLists.txt

@@ -8,6 +8,7 @@ add_subdirectory(func)
 add_subdirectory(mnode)
 add_subdirectory(profile)
 add_subdirectory(qnode)
+add_subdirectory(sdb)
 add_subdirectory(show)
 add_subdirectory(sma)
 add_subdirectory(snode)