Merge branch '3.0' into feature/TD-11381-3.0

.gitmodules

 [submodule "src/connector/go"]
 	path = src/connector/go
 	url = git@github.com:taosdata/driver-go.git
-[submodule "src/connector/grafanaplugin"]
-	path = src/connector/grafanaplugin
-	url = git@github.com:taosdata/grafanaplugin.git
 [submodule "src/connector/hivemq-tdengine-extension"]
 	path = src/connector/hivemq-tdengine-extension
 	url = git@github.com:taosdata/hivemq-tdengine-extension.git
-[submodule "tests/examples/rust"]
-	path = tests/examples/rust
-	url = https://github.com/songtianyi/tdengine-rust-bindings.git
 [submodule "deps/jemalloc"]
 	path = deps/jemalloc
 	url = https://github.com/jemalloc/jemalloc
 [submodule "deps/TSZ"]
 	path = deps/TSZ
 	url = https://github.com/taosdata/TSZ.git
+[submodule "tests"]
+	path = tests
+	url = https://github.com/taosdata/tests
+	branch = 3.0

tests/parallel_test/Jenkinsfile → Jenkinsfile2

@@ -74,6 +74,38 @@ def pre_test(){
     git pull >/dev/null
     git fetch origin +refs/pull/${CHANGE_ID}/merge
     git checkout -qf FETCH_HEAD
+    git submodule update --init --recursive --remote
+    '''
+    script {
+      if (env.CHANGE_TARGET == 'master') {
+        sh '''
+        cd ${WKCT}
+        git checkout master
+        '''
+        }
+      else if(env.CHANGE_TARGET == '2.0'){
+        sh '''
+        cd ${WKCT}
+        git checkout 2.0
+        '''
+      }
+      else if(env.CHANGE_TARGET == '3.0'){
+        sh '''
+        cd ${WKCT}
+        git checkout 3.0
+        '''
+      } 
+      else{
+        sh '''
+        cd ${WKCT}
+        git checkout develop
+        '''
+      }
+    }
+    sh'''
+    cd ${WKCT}
+    git pull >/dev/null
+    cd ${WKC}
     export TZ=Asia/Harbin
     date
     rm -rf debug
@@ -81,7 +113,6 @@ def pre_test(){
     cd debug
     cmake .. > /dev/null
     make -j4> /dev/null
-
     '''
     return 1
 }
@@ -92,6 +123,7 @@ pipeline {
   environment{
       WK = '/var/lib/jenkins/workspace/TDinternal'
       WKC= '/var/lib/jenkins/workspace/TDengine'
+      WKCT= '/var/lib/jenkins/workspace/TDengine/tests'
   }
   stages {
       stage('pre_build'){

include/common/tmsg.h

@@ -111,8 +111,8 @@ typedef enum _mgmt_table {
   TSDB_MGMT_TABLE_MAX,
 } EShowType;
 
-#define TSDB_ALTER_TABLE_ADD_TAG_COLUMN 1
-#define TSDB_ALTER_TABLE_DROP_TAG_COLUMN 2
+#define TSDB_ALTER_TABLE_ADD_TAG 1
+#define TSDB_ALTER_TABLE_DROP_TAG 2
 #define TSDB_ALTER_TABLE_UPDATE_TAG_NAME 3
 #define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4
 
@@ -163,6 +163,12 @@ typedef struct {
   int32_t vgVersion;
 } SBuildUseDBInput;
 
+typedef struct SField {
+  char    name[TSDB_COL_NAME_LEN];
+  uint8_t type;
+  int32_t bytes;
+} SField;
+
 #pragma pack(push, 1)
 
 // null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta
@@ -252,23 +258,34 @@ typedef struct SSchema {
 typedef struct {
   char    name[TSDB_TABLE_FNAME_LEN];
   int8_t  igExists;
-  int32_t numOfTags;
   int32_t numOfColumns;
-  SSchema pSchemas[];
+  int32_t numOfTags;
+  SArray* pColumns;
+  SArray* pTags;
+  char    comment[TSDB_STB_COMMENT_LEN];
 } SMCreateStbReq;
 
+int32_t tSerializeSMCreateStbReq(void** buf, SMCreateStbReq* pReq);
+void*   tDeserializeSMCreateStbReq(void* buf, SMCreateStbReq* pReq);
+
 typedef struct {
   char   name[TSDB_TABLE_FNAME_LEN];
   int8_t igNotExists;
 } SMDropStbReq;
 
+int32_t tSerializeSMDropStbReq(void** buf, SMDropStbReq* pReq);
+void*   tDeserializeSMDropStbReq(void* buf, SMDropStbReq* pReq);
+
 typedef struct {
   char    name[TSDB_TABLE_FNAME_LEN];
   int8_t  alterType;
-  int32_t numOfSchemas;
-  SSchema pSchemas[];
+  int32_t numOfFields;
+  SArray* pFields;
 } SMAltertbReq;
 
+int32_t tSerializeSMAlterStbReq(void** buf, SMAltertbReq* pReq);
+void*   tDeserializeSMAlterStbReq(void* buf, SMAltertbReq* pReq);
+
 typedef struct {
   int32_t pid;
   char    app[TSDB_APP_NAME_LEN];
@@ -597,7 +614,6 @@ typedef struct {
 typedef struct {
   int32_t vgId;
   int8_t  role;
-  int8_t  align[3];
   int64_t totalStorage;
   int64_t compStorage;
   int64_t pointsWritten;
@@ -605,27 +621,22 @@ typedef struct {
 } SVnodeLoad;
 
 typedef struct {
-  int32_t    num;
-  SVnodeLoad data[];
-} SVnodeLoads;
-
-typedef struct {
-  int32_t     sver;
+  int32_t     mver;  // msg version
+  int32_t     sver;  // software version
+  int64_t     dver;  // dnode table version in sdb
   int32_t     dnodeId;
   int64_t     clusterId;
-  int64_t     dver;
   int64_t     rebootTime;
   int64_t     updateTime;
   int32_t     numOfCores;
   int32_t     numOfSupportVnodes;
   char        dnodeEp[TSDB_EP_LEN];
   SClusterCfg clusterCfg;
-  SVnodeLoads vnodeLoads;
+  SArray*     pVloads;  // array of SVnodeLoad
 } SStatusReq;
 
-typedef struct {
-  int32_t reserved;
-} STransReq;
+int32_t tSerializeSStatusReq(void** buf, SStatusReq* pReq);
+void*   tDeserializeSStatusReq(void* buf, SStatusReq* pReq);
 
 typedef struct {
   int32_t dnodeId;
@@ -635,21 +646,23 @@ typedef struct {
 typedef struct {
   int32_t  id;
   int8_t   isMnode;
-  int8_t   align;
   SEp      ep;
 } SDnodeEp;
 
 typedef struct {
-  int32_t  num;
-  SDnodeEp eps[];
-} SDnodeEps;
-
-typedef struct {
+  int32_t   mver;
   int64_t   dver;
   SDnodeCfg dnodeCfg;
-  SDnodeEps dnodeEps;
+  SArray*   pDnodeEps;  // Array of SDnodeEp
 } SStatusRsp;
 
+int32_t tSerializeSStatusRsp(void** buf, SStatusRsp* pRsp);
+void*   tDeserializeSStatusRsp(void* buf, SStatusRsp* pRsp);
+
+typedef struct {
+  int32_t mver;
+} STransReq;
+
 typedef struct {
   int32_t  id;
   uint16_t port;                 // node sync Port
@@ -726,6 +739,7 @@ typedef struct {
   char     tbName[TSDB_TABLE_NAME_LEN];
   char     stbName[TSDB_TABLE_NAME_LEN];
   char     dbFName[TSDB_DB_FNAME_LEN];
+  uint64_t dbId;
   int32_t  numOfTags;
   int32_t  numOfColumns;
   int8_t   precision;
@@ -1110,10 +1124,14 @@ static FORCE_INLINE void* tDeserializeSMVSubscribeReq(void* buf, SMVSubscribeReq
   return buf;
 }
 
-typedef struct SMqTmrMsg {
+typedef struct {
   int32_t reserved;
 } SMqTmrMsg;
 
+typedef struct {
+  int64_t consumerId;
+} SMqDoRebalanceMsg;
+
 typedef struct {
   int64_t status;
 } SMVSubscribeRsp;
@@ -1185,8 +1203,6 @@ typedef struct {
 
 int32_t tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq);
 void*   tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq);
-int32_t tSerializeSVCreateTbRsp(void** buf, SVCreateTbRsp* pRsp);
-void*   tDeserializeSVCreateTbRsp(void* buf, SVCreateTbRsp* pRsp);
 
 typedef struct {
   uint64_t ver;  // use a general definition
@@ -1198,8 +1214,6 @@ typedef struct {
 
 int32_t tSerializeSVCreateTbBatchReq(void** buf, SVCreateTbBatchReq* pReq);
 void*   tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pReq);
-int32_t tSerializeSVCreateTbBatchReqp(void** buf, SVCreateTbBatchReq* pRsp);
-void*   tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pRsp);
 
 typedef struct {
   uint64_t ver;
@@ -1209,13 +1223,10 @@ typedef struct {
 } SVDropTbReq;
 
 typedef struct {
-  uint64_t ver;
 } SVDropTbRsp;
 
 int32_t tSerializeSVDropTbReq(void** buf, SVDropTbReq* pReq);
 void*   tDeserializeSVDropTbReq(void* buf, SVDropTbReq* pReq);
-int32_t tSerializeSVDropTbRsp(void** buf, SVDropTbRsp* pRsp);
-void*   tDeserializeSVDropTbRsp(void* buf, SVDropTbRsp* pRsp);
 
 typedef struct {
   SMsgHead head;
@@ -1691,13 +1702,13 @@ static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW)
   return buf;
 }
 
-typedef struct SMqTbData {
+typedef struct {
   int64_t    uid;
   int32_t    numOfRows;
   char*      colData;
 } SMqTbData;
 
-typedef struct SMqTopicBlk {
+typedef struct {
   char       topicName[TSDB_TOPIC_FNAME_LEN];
   int64_t    committedOffset;
   int64_t    reqOffset;
@@ -1708,7 +1719,7 @@ typedef struct SMqTopicBlk {
   SMqTbData* tbData;
 } SMqTopicData;
 
-typedef struct SMqConsumeRsp {
+typedef struct {
   int64_t         consumerId;
   SSchemaWrapper* schemas;
   int64_t         committedOffset;
@@ -1720,7 +1731,7 @@ typedef struct SMqConsumeRsp {
 } SMqConsumeRsp;
 
 // one req for one vg+topic
-typedef struct SMqConsumeReq {
+typedef struct {
   SMsgHead       head;
   //0: commit only, current offset
   //1: consume only, poll next offset
@@ -1736,17 +1747,17 @@ typedef struct SMqConsumeReq {
   char           topic[TSDB_TOPIC_FNAME_LEN];
 } SMqConsumeReq;
 
-typedef struct SMqSubVgEp {
+typedef struct {
   int32_t vgId;
   SEpSet  epSet;
 } SMqSubVgEp;
 
-typedef struct SMqSubTopicEp {
+typedef struct {
   char    topic[TSDB_TOPIC_FNAME_LEN];
   SArray* vgs;   // SArray<SMqSubVgEp>
 } SMqSubTopicEp;
 
-typedef struct SMqCMGetSubEpRsp {
+typedef struct {
   int64_t consumerId;
   int64_t epoch;
   char    cgroup[TSDB_CONSUMER_GROUP_LEN];

include/common/tmsgdef.h

@@ -141,7 +141,8 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp)
-  TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-timer", SMqTmrMsg, SMqTmrMsg)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-timer", SMqTmrMsg, SMqTmrMsg)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg)
 
   // Requests handled by VNODE
   TD_NEW_MSG_SEG(TDMT_VND_MSG)

include/dnode/mnode/mnode.h

@@ -27,6 +27,7 @@ typedef struct SMnodeMsg SMnodeMsg;
 typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg);
 typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg);
 typedef int32_t (*PutReqToMWriteQFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg);
+typedef int32_t (*PutReqToMReadQFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg);
 typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg);
 
 typedef struct SMnodeLoad {
@@ -64,6 +65,7 @@ typedef struct {
   SMnodeCfg         cfg;
   SDnode           *pDnode;
   PutReqToMWriteQFp putReqToMWriteQFp;
+  PutReqToMReadQFp  putReqToMReadQFp;
   SendReqToDnodeFp  sendReqToDnodeFp;
   SendReqToMnodeFp  sendReqToMnodeFp;
   SendRedirectRspFp sendRedirectRspFp;

include/libs/catalog/catalog.h

@@ -32,6 +32,15 @@ extern "C" {
 
 struct SCatalog;
 
+enum {
+  CTG_DBG_DB_NUM = 1,
+  CTG_DBG_META_NUM,
+  CTG_DBG_STB_NUM,
+  CTG_DBG_DB_RENT_NUM,
+  CTG_DBG_STB_RENT_NUM,
+};
+
+
 typedef struct SCatalogReq {
   SArray *pTableName;     // element is SNAME
   SArray *pUdf;           // udf name
@@ -99,7 +108,7 @@ int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName,
  */
 int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const char* pDBName, bool forceUpdate, SArray** pVgroupList);
 
-int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo);
+int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, uint64_t dbId, SDBVgroupInfo* dbInfo);
 
 int32_t catalogRemoveDB(struct SCatalog* pCatalog, const char* dbName, uint64_t dbId);
 
@@ -127,6 +136,8 @@ int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void * pTransporter, cons
  */
 int32_t catalogGetSTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta);
 
+int32_t catalogUpdateSTableMeta(struct SCatalog* pCatalog, STableMetaRsp *rspMsg);
+
 
 /**
  * Force renew a table's local cached meta data. 

include/libs/parser/parsenodes.h

@@ -37,12 +37,6 @@ typedef struct SQueryNode {
 
 #define queryNodeType(nodeptr) (((const SQueryNode*)(nodeptr))->type)
 
-typedef struct SField {
-  char     name[TSDB_COL_NAME_LEN];
-  uint8_t  type;
-  int32_t  bytes;
-} SField;
-
 typedef struct SFieldInfo {
   int16_t     numOfOutput;   // number of column in result
   SField     *final;

include/libs/qcom/query.h

@@ -81,7 +81,6 @@ typedef struct STableMeta {
 } STableMeta;
 
 typedef struct SDBVgroupInfo {
-  uint64_t  dbId;
   int32_t   vgVersion;  
   int8_t    hashMethod;
   SHashObj *vgHash;  //key:vgId, value:SVgroupInfo
@@ -89,6 +88,7 @@ typedef struct SDBVgroupInfo {
 
 typedef struct SUseDbOutput {
   char           db[TSDB_DB_FNAME_LEN];
+  uint64_t       dbId;
   SDBVgroupInfo *dbVgroup;
 } SUseDbOutput;
 
@@ -102,6 +102,7 @@ enum {
 
 typedef struct STableMetaOutput {
   int32_t     metaType;
+  uint64_t    dbId;
   char        dbFName[TSDB_DB_FNAME_LEN];
   char        ctbName[TSDB_TABLE_NAME_LEN];
   char        tbName[TSDB_TABLE_NAME_LEN];
@@ -159,6 +160,8 @@ void initQueryModuleMsgHandle();
 const SSchema* tGetTbnameColumnSchema();
 bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags);
 
+int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta **pMeta);
+
 extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen);
 extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize);
 

include/nodes/nodes.h

@@ -55,7 +55,7 @@ typedef enum ENodeType {
   QUERY_NODE_FILL,
 
   // only for parser
-  QUERY_NODE_TARGET_EXPR,
+  QUERY_NODE_RAW_EXPR,
 
   QUERY_NODE_SET_OPERATOR,
   QUERY_NODE_SELECT_STMT,
@@ -81,6 +81,13 @@ typedef struct SNodeList {
   SListCell* pTail;
 } SNodeList;
 
+typedef struct SRawExprNode {
+  ENodeType nodeType;
+  char* p;
+  uint32_t n;
+  SNode* pNode;
+} SRawExprNode;
+
 typedef struct SDataType {
   uint8_t type;
   uint8_t precision;
@@ -114,6 +121,14 @@ typedef struct SColumnNode {
 typedef struct SValueNode {
   SExprNode node; // QUERY_NODE_VALUE
   char* literal;
+  bool isDuration;
+  union {
+    bool b;
+    int64_t i;
+    uint64_t u;
+    double d;
+    char* p;
+  } datum;
 } SValueNode;
 
 typedef enum EOperatorType {
@@ -174,7 +189,7 @@ typedef struct SNodeListNode {
 } SNodeListNode;
 
 typedef struct SFunctionNode {
-  SExprNode type; // QUERY_NODE_FUNCTION
+  SExprNode node; // QUERY_NODE_FUNCTION
   char functionName[TSDB_FUNC_NAME_LEN];
   int32_t funcId;
   SNodeList* pParameterList;

include/util/taoserror.h

@@ -218,18 +218,19 @@ int32_t* taosGetErrno();
 // mnode-stable
 #define TSDB_CODE_MND_STB_ALREADY_EXIST         TAOS_DEF_ERROR_CODE(0, 0x03A0)
 #define TSDB_CODE_MND_STB_NOT_EXIST             TAOS_DEF_ERROR_CODE(0, 0x03A1)
-#define TSDB_CODE_MND_TOO_MANY_STBS             TAOS_DEF_ERROR_CODE(0, 0x03A2)
-#define TSDB_CODE_MND_INVALID_STB               TAOS_DEF_ERROR_CODE(0, 0x03A3)
-#define TSDB_CODE_MND_INVALID_STB_OPTION        TAOS_DEF_ERROR_CODE(0, 0x03A4)
-#define TSDB_CODE_MND_STB_OPTION_UNCHNAGED      TAOS_DEF_ERROR_CODE(0, 0x03A5)
-#define TSDB_CODE_MND_TOO_MANY_TAGS             TAOS_DEF_ERROR_CODE(0, 0x03A6)
-#define TSDB_CODE_MND_TAG_ALREAY_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03A7)
-#define TSDB_CODE_MND_TAG_NOT_EXIST             TAOS_DEF_ERROR_CODE(0, 0x03A8)
-#define TSDB_CODE_MND_TOO_MANY_COLUMNS          TAOS_DEF_ERROR_CODE(0, 0x03A9)
-#define TSDB_CODE_MND_COLUMN_ALREAY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03AA)
-#define TSDB_CODE_MND_COLUMN_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03AB)
-#define TSDB_CODE_MND_INVALID_ROW_BYTES         TAOS_DEF_ERROR_CODE(0, 0x03AC)
-#define TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC  TAOS_DEF_ERROR_CODE(0, 0x03AD)
+#define TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC  TAOS_DEF_ERROR_CODE(0, 0x03A2)
+#define TSDB_CODE_MND_TOO_MANY_STBS             TAOS_DEF_ERROR_CODE(0, 0x03A3)
+#define TSDB_CODE_MND_INVALID_STB               TAOS_DEF_ERROR_CODE(0, 0x03A4)
+#define TSDB_CODE_MND_INVALID_STB_OPTION        TAOS_DEF_ERROR_CODE(0, 0x03A5)
+#define TSDB_CODE_MND_INVALID_STB_ALTER_OPTION  TAOS_DEF_ERROR_CODE(0, 0x03A6)
+#define TSDB_CODE_MND_STB_OPTION_UNCHNAGED      TAOS_DEF_ERROR_CODE(0, 0x03A7)
+#define TSDB_CODE_MND_INVALID_ROW_BYTES         TAOS_DEF_ERROR_CODE(0, 0x03A8)
+#define TSDB_CODE_MND_TOO_MANY_TAGS             TAOS_DEF_ERROR_CODE(0, 0x03A9)
+#define TSDB_CODE_MND_TAG_ALREADY_EXIST         TAOS_DEF_ERROR_CODE(0, 0x03AA)
+#define TSDB_CODE_MND_TAG_NOT_EXIST             TAOS_DEF_ERROR_CODE(0, 0x03AB)
+#define TSDB_CODE_MND_TOO_MANY_COLUMNS          TAOS_DEF_ERROR_CODE(0, 0x03AC)
+#define TSDB_CODE_MND_COLUMN_ALREADY_EXIST      TAOS_DEF_ERROR_CODE(0, 0x03AD)
+#define TSDB_CODE_MND_COLUMN_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03AE)
 
 // mnode-func
 #define TSDB_CODE_MND_FUNC_ALREADY_EXIST        TAOS_DEF_ERROR_CODE(0, 0x03C0)
@@ -435,16 +436,21 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_CTG_NOT_READY                 TAOS_DEF_ERROR_CODE(0, 0x2402)  //catalog is not ready
 #define TSDB_CODE_CTG_MEM_ERROR                 TAOS_DEF_ERROR_CODE(0, 0x2403)  //catalog memory error
 #define TSDB_CODE_CTG_SYS_ERROR                 TAOS_DEF_ERROR_CODE(0, 0x2404)  //catalog system error
+#define TSDB_CODE_CTG_DB_DROPPED                TAOS_DEF_ERROR_CODE(0, 0x2405)  //Database is dropped
+#define TSDB_CODE_CTG_OUT_OF_SERVICE            TAOS_DEF_ERROR_CODE(0, 0x2406)  //catalog is out of service
 
 //scheduler
 #define TSDB_CODE_SCH_STATUS_ERROR              TAOS_DEF_ERROR_CODE(0, 0x2501)  //scheduler status error
 #define TSDB_CODE_SCH_INTERNAL_ERROR            TAOS_DEF_ERROR_CODE(0, 0x2502)  //scheduler internal error
 
 //parser
-#define TSDB_CODE_PARSER_INVALID_COLUMN         TAOS_DEF_ERROR_CODE(0, 0x2601)  //invalid column name
-#define TSDB_CODE_PARSER_TABLE_NOT_EXIST        TAOS_DEF_ERROR_CODE(0, 0x2602)  //table not exist
-#define TSDB_CODE_PARSER_AMBIGUOUS_COLUMN       TAOS_DEF_ERROR_CODE(0, 0x2603)  //ambiguous column
-#define TSDB_CODE_PARSER_WRONG_VALUE_TYPE       TAOS_DEF_ERROR_CODE(0, 0x2604)  //wrong value type
+#define TSDB_CODE_PAR_INVALID_COLUMN            TAOS_DEF_ERROR_CODE(0, 0x2601)  //invalid column name
+#define TSDB_CODE_PAR_TABLE_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x2602)  //table not exist
+#define TSDB_CODE_PAR_AMBIGUOUS_COLUMN          TAOS_DEF_ERROR_CODE(0, 0x2603)  //ambiguous column
+#define TSDB_CODE_PAR_WRONG_VALUE_TYPE          TAOS_DEF_ERROR_CODE(0, 0x2604)  //wrong value type
+#define TSDB_CODE_PAR_FUNTION_PARA_NUM          TAOS_DEF_ERROR_CODE(0, 0x2605)  //invalid number of arguments
+#define TSDB_CODE_PAR_FUNTION_PARA_TYPE         TAOS_DEF_ERROR_CODE(0, 0x2606)  //inconsistent datatypes
+#define TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION  TAOS_DEF_ERROR_CODE(0, 0x2607)  //there mustn't be aggregation
 
 #ifdef __cplusplus
 }

include/util/tdef.h

@@ -186,8 +186,8 @@ do { \
 #define TSDB_MAX_SQL_SHOW_LEN     1024
 #define TSDB_MAX_ALLOWED_SQL_LEN  (1*1024*1024u)          // sql length should be less than 1mb
 
-#define TSDB_APP_NAME_LEN          TSDB_UNI_LEN
-
+#define TSDB_APP_NAME_LEN         TSDB_UNI_LEN
+#define TSDB_STB_COMMENT_LEN      1024
   /**
    *  In some scenarios uint16_t (0~65535) is used to store the row len.
    *  - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header.

source/client/src/clientHb.c

@@ -44,7 +44,6 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
       code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid);
     } else {
       SDBVgroupInfo vgInfo = {0};
-      vgInfo.dbId = rsp->uid;
       vgInfo.vgVersion = rsp->vgVersion;
       vgInfo.hashMethod = rsp->hashMethod;
       vgInfo.vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
@@ -69,7 +68,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
         }
       }  
       
-      code = catalogUpdateDBVgroup(pCatalog, rsp->db, &vgInfo);
+      code = catalogUpdateDBVgroup(pCatalog, rsp->db, rsp->uid, &vgInfo);
       if (code) {
         taosHashCleanup(vgInfo.vgHash);
       }
@@ -101,50 +100,33 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo
       
       tscDebug("hb remove stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName);
 
-      code = catalogRemoveSTableMeta(pCatalog, rsp->dbFName, rsp->stbName, rsp->suid);
+      catalogRemoveSTableMeta(pCatalog, rsp->dbFName, rsp->stbName, rsp->suid);
     } else {
+      tscDebug("hb update stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName);
+
       rsp->numOfTags = ntohl(rsp->numOfTags);
+      rsp->sversion = ntohl(rsp->sversion);
+      rsp->tversion = ntohl(rsp->tversion);
+      rsp->tuid = be64toh(rsp->tuid);
+      rsp->vgId = ntohl(rsp->vgId);
+
+      SSchema* pSchema = rsp->pSchema;
       
       schemaNum = rsp->numOfColumns + rsp->numOfTags;
-/*      
-      rsp->vgNum = ntohl(rsp->vgNum);
-      rsp->uid = be64toh(rsp->uid);
-
-      SDBVgroupInfo vgInfo = {0};
-      vgInfo.dbId = rsp->uid;
-      vgInfo.vgVersion = rsp->vgVersion;
-      vgInfo.hashMethod = rsp->hashMethod;
-      vgInfo.vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
-      if (NULL == vgInfo.vgHash) {
-        tscError("hash init[%d] failed", rsp->vgNum);
-        return TSDB_CODE_TSC_OUT_OF_MEMORY;
-      }
-
-      for (int32_t i = 0; i < rsp->vgNum; ++i) {
-        rsp->vgroupInfo[i].vgId = ntohl(rsp->vgroupInfo[i].vgId);
-        rsp->vgroupInfo[i].hashBegin = ntohl(rsp->vgroupInfo[i].hashBegin);
-        rsp->vgroupInfo[i].hashEnd = ntohl(rsp->vgroupInfo[i].hashEnd);
 
-        for (int32_t n = 0; n < rsp->vgroupInfo[i].epset.numOfEps; ++n) {
-          rsp->vgroupInfo[i].epset.eps[n].port = ntohs(rsp->vgroupInfo[i].epset.eps[n].port);
-        }
+      for (int i = 0; i < schemaNum; ++i) {
+        pSchema->bytes = ntohl(pSchema->bytes);
+        pSchema->colId = ntohl(pSchema->colId);
 
-        if (0 != taosHashPut(vgInfo.vgHash, &rsp->vgroupInfo[i].vgId, sizeof(rsp->vgroupInfo[i].vgId), &rsp->vgroupInfo[i], sizeof(rsp->vgroupInfo[i]))) {
-          tscError("hash push failed, errno:%d", errno);
-          taosHashCleanup(vgInfo.vgHash);
-          return TSDB_CODE_TSC_OUT_OF_MEMORY;
-        }
-      }  
-      
-      code = catalogUpdateDBVgroup(pCatalog, rsp->db, &vgInfo);
-      if (code) {
-        taosHashCleanup(vgInfo.vgHash);
+        pSchema++;
       }
-*/      
-    }
 
-    if (code) {
-      return code;
+      if (rsp->pSchema[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) {
+        tscError("invalid colId[%d] for the first column in table meta rsp msg", rsp->pSchema[0].colId);
+        return TSDB_CODE_TSC_INVALID_VALUE;
+      }      
+
+      catalogUpdateSTableMeta(pCatalog, rsp);
     }
 
     msgLen += sizeof(STableMetaRsp) + schemaNum * sizeof(SSchema);

source/client/src/tmq.c

@@ -209,6 +209,9 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
 
     SName name = {0};
     char* dbName = getDbOfConnection(tmq->pTscObj);
+    if (dbName == NULL) {
+      return TMQ_RESP_ERR__FAIL;
+    }
     tNameSetDbName(&name, tmq->pTscObj->acctId, dbName, strlen(dbName));
     tNameFromString(&name, topicName, T_NAME_TABLE);
 

source/client/test/clientTests.cpp

@@ -565,7 +565,6 @@ TEST(testCase, insert_test) {
 #endif
 
 
-#if 1
 TEST(testCase, projection_query_tables) {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   ASSERT_NE(pConn, nullptr);
@@ -591,7 +590,7 @@ TEST(testCase, projection_query_tables) {
   }
   taos_free_result(pRes);
 
-  for(int32_t i = 0; i < 100000; ++i) {
+  for(int32_t i = 0; i < 10000000; ++i) {
     char sql[512] = {0};
     sprintf(sql, "insert into tu values(now+%da, %d)", i, i);
     TAOS_RES* p = taos_query(pConn, sql);
@@ -622,6 +621,7 @@ TEST(testCase, projection_query_tables) {
   taos_free_result(pRes);
   taos_close(pConn);
 }
+#if 0
 
 TEST(testCase, projection_query_stables) {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);

source/client/test/tmqTest.cpp

@@ -46,7 +46,6 @@ TEST(testCase, create_topic_ctb_Test) {
   if (taos_errno(pRes) != 0) {
     printf("error in use db, reason:%s\n", taos_errstr(pRes));
   }
-  //taos_free_result(pRes);
 
   TAOS_FIELD* pFields = taos_fetch_fields(pRes);
   ASSERT_TRUE(pFields == nullptr);

source/common/src/tmsg.c

@@ -27,7 +27,7 @@
 #undef TD_MSG_SEG_CODE_
 #include "tmsgdef.h"
 
-int tInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) {
+int32_t tInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) {
   if (pMsg == NULL) {
     terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
     return -1;
@@ -44,7 +44,7 @@ int tInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) {
   return 0;
 }
 
-int tGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) {
+int32_t tGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) {
   if (pIter->len == 0) {
     pIter->len += sizeof(SSubmitMsg);
   } else {
@@ -63,7 +63,7 @@ int tGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) {
   return 0;
 }
 
-int tInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) {
+int32_t tInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) {
   if (pBlock->dataLen <= 0) return -1;
   pIter->totalLen = pBlock->dataLen;
   pIter->len = 0;
@@ -85,14 +85,14 @@ STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) {
   }
 }
 
-int tSerializeSClientHbReq(void **buf, const SClientHbReq *pReq) {
-  int tlen = 0;
+int32_t tSerializeSClientHbReq(void **buf, const SClientHbReq *pReq) {
+  int32_t tlen = 0;
   tlen += taosEncodeSClientHbKey(buf, &pReq->connKey);
 
   int32_t kvNum = taosHashGetSize(pReq->info);
   tlen += taosEncodeFixedI32(buf, kvNum);
-  SKv *kv;
-  void* pIter = taosHashIterate(pReq->info, NULL);
+  SKv  *kv;
+  void *pIter = taosHashIterate(pReq->info, NULL);
   while (pIter != NULL) {
     kv = pIter;
     tlen += taosEncodeSKv(buf, kv);
@@ -111,7 +111,7 @@ void *tDeserializeSClientHbReq(void *buf, SClientHbReq *pReq) {
   if (pReq->info == NULL) {
     pReq->info = taosHashInit(kvNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
   }
-  for(int i = 0; i < kvNum; i++) {
+  for (int32_t i = 0; i < kvNum; i++) {
     SKv kv;
     buf = taosDecodeSKv(buf, &kv);
     taosHashPut(pReq->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv));
@@ -120,54 +120,55 @@ void *tDeserializeSClientHbReq(void *buf, SClientHbReq *pReq) {
   return buf;
 }
 
-int   tSerializeSClientHbRsp(void** buf, const SClientHbRsp* pRsp) {
-  int tlen = 0;
+int32_t tSerializeSClientHbRsp(void **buf, const SClientHbRsp *pRsp) {
+  int32_t tlen = 0;
   int32_t kvNum = taosArrayGetSize(pRsp->info);
   tlen += taosEncodeSClientHbKey(buf, &pRsp->connKey);
   tlen += taosEncodeFixedI32(buf, pRsp->status);
   tlen += taosEncodeFixedI32(buf, kvNum);
-  for (int i = 0; i < kvNum; i++) {
+  for (int32_t i = 0; i < kvNum; i++) {
     SKv *kv = (SKv *)taosArrayGet(pRsp->info, i);
     tlen += taosEncodeSKv(buf, kv);
   }
   return tlen;
 }
-void* tDeserializeSClientHbRsp(void* buf, SClientHbRsp* pRsp) {
+
+void *tDeserializeSClientHbRsp(void *buf, SClientHbRsp *pRsp) {
   int32_t kvNum = 0;
   buf = taosDecodeSClientHbKey(buf, &pRsp->connKey);
   buf = taosDecodeFixedI32(buf, &pRsp->status);
   buf = taosDecodeFixedI32(buf, &kvNum);
   pRsp->info = taosArrayInit(kvNum, sizeof(SKv));
-  for (int i = 0; i < kvNum; i++) {
+  for (int32_t i = 0; i < kvNum; i++) {
     SKv kv = {0};
     buf = taosDecodeSKv(buf, &kv);
     taosArrayPush(pRsp->info, &kv);
   }
-  
+
   return buf;
 }
 
-int   tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pBatchReq) {
-  int tlen = 0;
+int32_t tSerializeSClientHbBatchReq(void **buf, const SClientHbBatchReq *pBatchReq) {
+  int32_t tlen = 0;
   tlen += taosEncodeFixedI64(buf, pBatchReq->reqId);
   int32_t reqNum = taosArrayGetSize(pBatchReq->reqs);
-  tlen += taosEncodeFixedI32(buf, reqNum); 
-  for (int i = 0; i < reqNum; i++) {
-    SClientHbReq* pReq = taosArrayGet(pBatchReq->reqs, i);
+  tlen += taosEncodeFixedI32(buf, reqNum);
+  for (int32_t i = 0; i < reqNum; i++) {
+    SClientHbReq *pReq = taosArrayGet(pBatchReq->reqs, i);
     tlen += tSerializeSClientHbReq(buf, pReq);
   }
   return tlen;
 }
 
-void* tDeserializeSClientHbBatchReq(void* buf, SClientHbBatchReq* pBatchReq) {
+void *tDeserializeSClientHbBatchReq(void *buf, SClientHbBatchReq *pBatchReq) {
   buf = taosDecodeFixedI64(buf, &pBatchReq->reqId);
   if (pBatchReq->reqs == NULL) {
     pBatchReq->reqs = taosArrayInit(0, sizeof(SClientHbReq));
   }
- 
+
   int32_t reqNum;
   buf = taosDecodeFixedI32(buf, &reqNum);
-  for (int i = 0; i < reqNum; i++) {
+  for (int32_t i = 0; i < reqNum; i++) {
     SClientHbReq req = {0};
     buf = tDeserializeSClientHbReq(buf, &req);
     taosArrayPush(pBatchReq->reqs, &req);
@@ -175,31 +176,31 @@ void* tDeserializeSClientHbBatchReq(void* buf, SClientHbBatchReq* pBatchReq) {
   return buf;
 }
 
-int   tSerializeSClientHbBatchRsp(void** buf, const SClientHbBatchRsp* pBatchRsp) {
-  int tlen = 0;
+int32_t tSerializeSClientHbBatchRsp(void **buf, const SClientHbBatchRsp *pBatchRsp) {
+  int32_t tlen = 0;
   int32_t sz = taosArrayGetSize(pBatchRsp->rsps);
   tlen += taosEncodeFixedI32(buf, sz);
-  for (int i = 0; i < sz; i++) {
-    SClientHbRsp* pRsp = taosArrayGet(pBatchRsp->rsps, i);
+  for (int32_t i = 0; i < sz; i++) {
+    SClientHbRsp *pRsp = taosArrayGet(pBatchRsp->rsps, i);
     tlen += tSerializeSClientHbRsp(buf, pRsp);
   }
   return tlen;
 }
 
-void* tDeserializeSClientHbBatchRsp(void* buf, SClientHbBatchRsp* pBatchRsp) {
+void *tDeserializeSClientHbBatchRsp(void *buf, SClientHbBatchRsp *pBatchRsp) {
   int32_t sz;
   buf = taosDecodeFixedI32(buf, &sz);
   pBatchRsp->rsps = taosArrayInit(sz, sizeof(SClientHbRsp));
-  for (int i = 0; i < sz; i++) {
+  for (int32_t i = 0; i < sz; i++) {
     SClientHbRsp rsp = {0};
-    buf = tDeserializeSClientHbRsp(buf, &rsp); 
+    buf = tDeserializeSClientHbRsp(buf, &rsp);
     taosArrayPush(pBatchRsp->rsps, &rsp);
   }
   return buf;
 }
 
-int tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
-  int tlen = 0;
+int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
+  int32_t tlen = 0;
 
   tlen += taosEncodeFixedU64(buf, pReq->ver);
   tlen += taosEncodeString(buf, pReq->name);
@@ -293,8 +294,8 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
   return buf;
 }
 
-int tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) {
-  int tlen = 0;
+int32_t tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) {
+  int32_t tlen = 0;
 
   tlen += taosEncodeFixedU64(buf, pReq->ver);
   tlen += taosEncodeFixedU32(buf, taosArrayGetSize(pReq->pArray));
@@ -322,7 +323,7 @@ void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) {
 }
 
 int32_t tSerializeSVDropTbReq(void **buf, SVDropTbReq *pReq) {
-  int tlen = 0;
+  int32_t tlen = 0;
   tlen += taosEncodeFixedU64(buf, pReq->ver);
   tlen += taosEncodeString(buf, pReq->name);
   tlen += taosEncodeFixedU8(buf, pReq->type);
@@ -335,3 +336,268 @@ void *tDeserializeSVDropTbReq(void *buf, SVDropTbReq *pReq) {
   buf = taosDecodeFixedU8(buf, &pReq->type);
   return buf;
 }
+
+int32_t tSerializeSMCreateStbReq(void **buf, SMCreateStbReq *pReq) {
+  int32_t tlen = 0;
+
+  tlen += taosEncodeString(buf, pReq->name);
+  tlen += taosEncodeFixedI8(buf, pReq->igExists);
+  tlen += taosEncodeFixedI32(buf, pReq->numOfColumns);
+  tlen += taosEncodeFixedI32(buf, pReq->numOfTags);
+
+  for (int32_t i = 0; i < pReq->numOfColumns; ++i) {
+    SField *pField = taosArrayGet(pReq->pColumns, i);
+    tlen += taosEncodeFixedI8(buf, pField->type);
+    tlen += taosEncodeFixedI32(buf, pField->bytes);
+    tlen += taosEncodeString(buf, pField->name);
+  }
+
+  for (int32_t i = 0; i < pReq->numOfTags; ++i) {
+    SField *pField = taosArrayGet(pReq->pTags, i);
+    tlen += taosEncodeFixedI8(buf, pField->type);
+    tlen += taosEncodeFixedI32(buf, pField->bytes);
+    tlen += taosEncodeString(buf, pField->name);
+  }
+
+  tlen += taosEncodeString(buf, pReq->comment);
+  return tlen;
+}
+
+void *tDeserializeSMCreateStbReq(void *buf, SMCreateStbReq *pReq) {
+  buf = taosDecodeStringTo(buf, pReq->name);
+  buf = taosDecodeFixedI8(buf, &pReq->igExists);
+  buf = taosDecodeFixedI32(buf, &pReq->numOfColumns);
+  buf = taosDecodeFixedI32(buf, &pReq->numOfTags);
+
+  pReq->pColumns = taosArrayInit(pReq->numOfColumns, sizeof(SField));
+  pReq->pTags = taosArrayInit(pReq->numOfTags, sizeof(SField));
+  if (pReq->pColumns == NULL || pReq->pTags == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  for (int32_t i = 0; i < pReq->numOfColumns; ++i) {
+    SField field = {0};
+    buf = taosDecodeFixedI8(buf, &field.type);
+    buf = taosDecodeFixedI32(buf, &field.bytes);
+    buf = taosDecodeStringTo(buf, field.name);
+    if (taosArrayPush(pReq->pColumns, &field) == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return NULL;
+    }
+  }
+
+  for (int32_t i = 0; i < pReq->numOfTags; ++i) {
+    SField field = {0};
+    buf = taosDecodeFixedI8(buf, &field.type);
+    buf = taosDecodeFixedI32(buf, &field.bytes);
+    buf = taosDecodeStringTo(buf, field.name);
+    if (taosArrayPush(pReq->pTags, &field) == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return NULL;
+    }
+  }
+
+  buf = taosDecodeStringTo(buf, pReq->comment);
+  return buf;
+}
+
+int32_t tSerializeSMDropStbReq(void **buf, SMDropStbReq *pReq) {
+  int32_t tlen = 0;
+
+  tlen += taosEncodeString(buf, pReq->name);
+  tlen += taosEncodeFixedI8(buf, pReq->igNotExists);
+
+  return tlen;
+}
+
+void *tDeserializeSMDropStbReq(void *buf, SMDropStbReq *pReq) {
+  buf = taosDecodeStringTo(buf, pReq->name);
+  buf = taosDecodeFixedI8(buf, &pReq->igNotExists);
+
+  return buf;
+}
+
+int32_t tSerializeSMAlterStbReq(void **buf, SMAltertbReq *pReq) {
+  int32_t tlen = 0;
+
+  tlen += taosEncodeString(buf, pReq->name);
+  tlen += taosEncodeFixedI8(buf, pReq->alterType);
+  tlen += taosEncodeFixedI32(buf, pReq->numOfFields);
+
+  for (int32_t i = 0; i < pReq->numOfFields; ++i) {
+    SField *pField = taosArrayGet(pReq->pFields, i);
+    tlen += taosEncodeFixedU8(buf, pField->type);
+    tlen += taosEncodeFixedI32(buf, pField->bytes);
+    tlen += taosEncodeString(buf, pField->name);
+  }
+
+  return tlen;
+}
+
+void *tDeserializeSMAlterStbReq(void *buf, SMAltertbReq *pReq) {
+  buf = taosDecodeStringTo(buf, pReq->name);
+  buf = taosDecodeFixedI8(buf, &pReq->alterType);
+  buf = taosDecodeFixedI32(buf, &pReq->numOfFields);
+
+  pReq->pFields = taosArrayInit(pReq->numOfFields, sizeof(SField));
+  if (pReq->pFields == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  for (int32_t i = 0; i < pReq->numOfFields; ++i) {
+    SField field = {0};
+    buf = taosDecodeFixedU8(buf, &field.type);
+    buf = taosDecodeFixedI32(buf, &field.bytes);
+    buf = taosDecodeStringTo(buf, field.name);
+    if (taosArrayPush(pReq->pFields, &field) == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return NULL;
+    }
+  }
+
+  return buf;
+}
+
+int32_t tSerializeSStatusReq(void **buf, SStatusReq *pReq) {
+  int32_t tlen = 0;
+
+  // status
+  tlen += taosEncodeFixedI32(buf, pReq->mver);
+  tlen += taosEncodeFixedI32(buf, pReq->sver);
+  tlen += taosEncodeFixedI64(buf, pReq->dver);
+  tlen += taosEncodeFixedI32(buf, pReq->dnodeId);
+  tlen += taosEncodeFixedI64(buf, pReq->clusterId);
+  tlen += taosEncodeFixedI64(buf, pReq->rebootTime);
+  tlen += taosEncodeFixedI64(buf, pReq->updateTime);
+  tlen += taosEncodeFixedI32(buf, pReq->numOfCores);
+  tlen += taosEncodeFixedI32(buf, pReq->numOfSupportVnodes);
+  tlen += taosEncodeString(buf, pReq->dnodeEp);
+
+  // cluster cfg
+  tlen += taosEncodeFixedI32(buf, pReq->clusterCfg.statusInterval);
+  tlen += taosEncodeFixedI64(buf, pReq->clusterCfg.checkTime);
+  tlen += taosEncodeString(buf, pReq->clusterCfg.timezone);
+  tlen += taosEncodeString(buf, pReq->clusterCfg.locale);
+  tlen += taosEncodeString(buf, pReq->clusterCfg.charset);
+
+  // vnode loads
+  int32_t vlen = (int32_t)taosArrayGetSize(pReq->pVloads);
+  tlen += taosEncodeFixedI32(buf, vlen);
+  for (int32_t i = 0; i < vlen; ++i) {
+    SVnodeLoad *pload = taosArrayGet(pReq->pVloads, i);
+    tlen += taosEncodeFixedI32(buf, pload->vgId);
+    tlen += taosEncodeFixedI8(buf, pload->role);
+    tlen += taosEncodeFixedI64(buf, pload->totalStorage);
+    tlen += taosEncodeFixedI64(buf, pload->compStorage);
+    tlen += taosEncodeFixedI64(buf, pload->pointsWritten);
+    tlen += taosEncodeFixedI64(buf, pload->tablesNum);
+  }
+
+  return tlen;
+}
+
+void *tDeserializeSStatusReq(void *buf, SStatusReq *pReq) {
+  // status
+  buf = taosDecodeFixedI32(buf, &pReq->mver);
+  buf = taosDecodeFixedI32(buf, &pReq->sver);
+  buf = taosDecodeFixedI64(buf, &pReq->dver);
+  buf = taosDecodeFixedI32(buf, &pReq->dnodeId);
+  buf = taosDecodeFixedI64(buf, &pReq->clusterId);
+  buf = taosDecodeFixedI64(buf, &pReq->rebootTime);
+  buf = taosDecodeFixedI64(buf, &pReq->updateTime);
+  buf = taosDecodeFixedI32(buf, &pReq->numOfCores);
+  buf = taosDecodeFixedI32(buf, &pReq->numOfSupportVnodes);
+  buf = taosDecodeStringTo(buf, pReq->dnodeEp);
+
+  // cluster cfg
+  buf = taosDecodeFixedI32(buf, &pReq->clusterCfg.statusInterval);
+  buf = taosDecodeFixedI64(buf, &pReq->clusterCfg.checkTime);
+  buf = taosDecodeStringTo(buf, pReq->clusterCfg.timezone);
+  buf = taosDecodeStringTo(buf, pReq->clusterCfg.locale);
+  buf = taosDecodeStringTo(buf, pReq->clusterCfg.charset);
+
+  // vnode loads
+  int32_t vlen = 0;
+  buf = taosDecodeFixedI32(buf, &vlen);
+  pReq->pVloads = taosArrayInit(vlen, sizeof(SVnodeLoad));
+  if (pReq->pVloads == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  for (int32_t i = 0; i < vlen; ++i) {
+    SVnodeLoad vload = {0};
+    buf = taosDecodeFixedI32(buf, &vload.vgId);
+    buf = taosDecodeFixedI8(buf, &vload.role);
+    buf = taosDecodeFixedI64(buf, &vload.totalStorage);
+    buf = taosDecodeFixedI64(buf, &vload.compStorage);
+    buf = taosDecodeFixedI64(buf, &vload.pointsWritten);
+    buf = taosDecodeFixedI64(buf, &vload.tablesNum);
+    if (taosArrayPush(pReq->pVloads, &vload) == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return NULL;
+    }
+  }
+
+  return buf;
+}
+
+int32_t tSerializeSStatusRsp(void **buf, SStatusRsp *pRsp) {
+  int32_t tlen = 0;
+
+  // status
+  tlen += taosEncodeFixedI32(buf, pRsp->mver);
+  tlen += taosEncodeFixedI64(buf, pRsp->dver);
+
+  // dnode cfg
+  tlen += taosEncodeFixedI32(buf, pRsp->dnodeCfg.dnodeId);
+  tlen += taosEncodeFixedI64(buf, pRsp->dnodeCfg.clusterId);
+
+  // dnode eps
+  int32_t dlen = (int32_t)taosArrayGetSize(pRsp->pDnodeEps);
+  tlen += taosEncodeFixedI32(buf, dlen);
+  for (int32_t i = 0; i < dlen; ++i) {
+    SDnodeEp *pDnodeEp = taosArrayGet(pRsp->pDnodeEps, i);
+    tlen += taosEncodeFixedI32(buf, pDnodeEp->id);
+    tlen += taosEncodeFixedI8(buf, pDnodeEp->isMnode);
+    tlen += taosEncodeString(buf, pDnodeEp->ep.fqdn);
+    tlen += taosEncodeFixedU16(buf, pDnodeEp->ep.port);
+  }
+
+  return tlen;
+}
+
+void *tDeserializeSStatusRsp(void *buf, SStatusRsp *pRsp) {
+  // status
+  buf = taosDecodeFixedI32(buf, &pRsp->mver);
+  buf = taosDecodeFixedI64(buf, &pRsp->dver);
+
+  // cluster cfg
+  buf = taosDecodeFixedI32(buf, &pRsp->dnodeCfg.dnodeId);
+  buf = taosDecodeFixedI64(buf, &pRsp->dnodeCfg.clusterId);
+
+  // dnode eps
+  int32_t dlen = 0;
+  buf = taosDecodeFixedI32(buf, &dlen);
+  pRsp->pDnodeEps = taosArrayInit(dlen, sizeof(SDnodeEp));
+  if (pRsp->pDnodeEps == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  for (int32_t i = 0; i < dlen; ++i) {
+    SDnodeEp dnodeEp = {0};
+    buf = taosDecodeFixedI32(buf, &dnodeEp.id);
+    buf = taosDecodeFixedI8(buf, &dnodeEp.isMnode);
+    buf = taosDecodeStringTo(buf, dnodeEp.ep.fqdn);
+    buf = taosDecodeFixedU16(buf, &dnodeEp.ep.port);
+    if (taosArrayPush(pRsp->pDnodeEps, &dnodeEp) == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return NULL;
+    }
+  }
+
+  return buf;
+}

source/dnode/mgmt/impl/inc/dndEnv.h

@@ -55,7 +55,7 @@ typedef struct {
   SEpSet       mnodeEpSet;
   char        *file;
   SHashObj    *dnodeHash;
-  SDnodeEps   *dnodeEps;
+  SArray      *pDnodeEps;
   pthread_t   *threadId;
   SRWLatch     latch;
   SDnodeWorker mgmtWorker;

source/dnode/mgmt/impl/inc/dndVnodes.h

@@ -23,7 +23,7 @@ extern "C" {
 
 int32_t dndInitVnodes(SDnode *pDnode);
 void    dndCleanupVnodes(SDnode *pDnode);
-void    dndGetVnodeLoads(SDnode *pDnode, SVnodeLoads *pVloads);
+void    dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads);
 void    dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
 void    dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
 void    dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);

source/dnode/mgmt/impl/src/dndMgmt.c

@@ -113,35 +113,31 @@ static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) {
 static void dndPrintDnodes(SDnode *pDnode) {
   SDnodeMgmt *pMgmt = &pDnode->dmgmt;
 
-  dDebug("print dnode ep list, num:%d", pMgmt->dnodeEps->num);
-  for (int32_t i = 0; i < pMgmt->dnodeEps->num; i++) {
-    SDnodeEp *pEp = &pMgmt->dnodeEps->eps[i];
+  int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->pDnodeEps);
+  dDebug("print dnode ep list, num:%d", numOfEps);
+  for (int32_t i = 0; i < numOfEps; i++) {
+    SDnodeEp *pEp = taosArrayGet(pMgmt->pDnodeEps, i);
     dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", pEp->id, pEp->ep.fqdn, pEp->ep.port, pEp->isMnode);
   }
 }
 
-static void dndResetDnodes(SDnode *pDnode, SDnodeEps *pDnodeEps) {
+static void dndResetDnodes(SDnode *pDnode, SArray *pDnodeEps) {
   SDnodeMgmt *pMgmt = &pDnode->dmgmt;
 
-  int32_t size = sizeof(SDnodeEps) + pDnodeEps->num * sizeof(SDnodeEp);
-  if (pDnodeEps->num > pMgmt->dnodeEps->num) {
-    SDnodeEps *tmp = calloc(1, size);
-    if (tmp == NULL) return;
-
-    tfree(pMgmt->dnodeEps);
-    pMgmt->dnodeEps = tmp;
-  }
-
-  if (pMgmt->dnodeEps != pDnodeEps) {
-    memcpy(pMgmt->dnodeEps, pDnodeEps, size);
+  if (pMgmt->pDnodeEps != pDnodeEps) {
+    SArray *tmp = pMgmt->pDnodeEps;
+    pMgmt->pDnodeEps = taosArrayDup(pDnodeEps);
+    taosArrayDestroy(tmp);
   }
 
   pMgmt->mnodeEpSet.inUse = 0;
   pMgmt->mnodeEpSet.numOfEps = 0;
 
   int32_t mIndex = 0;
-  for (int32_t i = 0; i < pMgmt->dnodeEps->num; i++) {
-    SDnodeEp *pDnodeEp = &pMgmt->dnodeEps->eps[i];
+  int32_t numOfEps = (int32_t)taosArrayGetSize(pDnodeEps);
+
+  for (int32_t i = 0; i < numOfEps; i++) {
+    SDnodeEp *pDnodeEp = taosArrayGet(pDnodeEps, i);
     if (!pDnodeEp->isMnode) continue;
     if (mIndex >= TSDB_MAX_REPLICA) continue;
     pMgmt->mnodeEpSet.numOfEps++;
@@ -150,8 +146,8 @@ static void dndResetDnodes(SDnode *pDnode, SDnodeEps *pDnodeEps) {
     mIndex++;
   }
 
-  for (int32_t i = 0; i < pMgmt->dnodeEps->num; ++i) {
-    SDnodeEp *pDnodeEp = &pMgmt->dnodeEps->eps[i];
+  for (int32_t i = 0; i < numOfEps; i++) {
+    SDnodeEp *pDnodeEp = taosArrayGet(pDnodeEps, i);
     taosHashPut(pMgmt->dnodeHash, &pDnodeEp->id, sizeof(int32_t), pDnodeEp, sizeof(SDnodeEp));
   }
 
@@ -178,6 +174,12 @@ static bool dndIsEpChanged(SDnode *pDnode, int32_t dnodeId, char *pEp) {
 static int32_t dndReadDnodes(SDnode *pDnode) {
   SDnodeMgmt *pMgmt = &pDnode->dmgmt;
 
+  pMgmt->pDnodeEps = taosArrayInit(1, sizeof(SDnodeEp));
+  if (pMgmt->pDnodeEps == NULL) {
+    dError("failed to calloc dnodeEp array since %s", strerror(errno));
+    goto PRASE_DNODE_OVER;
+  }
+
   int32_t code = TSDB_CODE_DND_DNODE_READ_FILE_ERROR;
   int32_t len = 0;
   int32_t maxLen = 256 * 1024;
@@ -238,18 +240,11 @@ static int32_t dndReadDnodes(SDnode *pDnode) {
     goto PRASE_DNODE_OVER;
   }
 
-  pMgmt->dnodeEps = calloc(1, numOfDnodes * sizeof(SDnodeEp) + sizeof(SDnodeEps));
-  if (pMgmt->dnodeEps == NULL) {
-    dError("failed to calloc dnodeEpList since %s", strerror(errno));
-    goto PRASE_DNODE_OVER;
-  }
-  pMgmt->dnodeEps->num = numOfDnodes;
-
   for (int32_t i = 0; i < numOfDnodes; ++i) {
     cJSON *node = cJSON_GetArrayItem(dnodes, i);
     if (node == NULL) break;
 
-    SDnodeEp *pDnodeEp = &pMgmt->dnodeEps->eps[i];
+    SDnodeEp dnodeEp = {0};
 
     cJSON *did = cJSON_GetObjectItem(node, "id");
     if (!did || did->type != cJSON_Number) {
@@ -257,14 +252,14 @@ static int32_t dndReadDnodes(SDnode *pDnode) {
       goto PRASE_DNODE_OVER;
     }
 
-    pDnodeEp->id = dnodeId->valueint;
+    dnodeEp.id = dnodeId->valueint;
 
     cJSON *dnodeFqdn = cJSON_GetObjectItem(node, "fqdn");
     if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) {
       dError("failed to read %s since dnodeFqdn not found", pMgmt->file);
       goto PRASE_DNODE_OVER;
     }
-    tstrncpy(pDnodeEp->ep.fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN);
+    tstrncpy(dnodeEp.ep.fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN);
 
     cJSON *dnodePort = cJSON_GetObjectItem(node, "port");
     if (!dnodePort || dnodePort->type != cJSON_Number) {
@@ -272,14 +267,16 @@ static int32_t dndReadDnodes(SDnode *pDnode) {
       goto PRASE_DNODE_OVER;
     }
 
-    pDnodeEp->ep.port = dnodePort->valueint;
+    dnodeEp.ep.port = dnodePort->valueint;
 
     cJSON *isMnode = cJSON_GetObjectItem(node, "isMnode");
     if (!isMnode || isMnode->type != cJSON_Number) {
       dError("failed to read %s since isMnode not found", pMgmt->file);
       goto PRASE_DNODE_OVER;
     }
-    pDnodeEp->isMnode = isMnode->valueint;
+    dnodeEp.isMnode = isMnode->valueint;
+
+    taosArrayPush(pMgmt->pDnodeEps, &dnodeEp);
   }
 
   code = 0;
@@ -296,15 +293,14 @@ PRASE_DNODE_OVER:
     return -1;
   }
 
-  if (pMgmt->dnodeEps == NULL) {
-    pMgmt->dnodeEps = calloc(1, sizeof(SDnodeEps) + sizeof(SDnodeEp));
-    pMgmt->dnodeEps->num = 1;
-    pMgmt->dnodeEps->eps[0].isMnode = 1;
-
-    taosGetFqdnPortFromEp(pDnode->cfg.firstEp, &(pMgmt->dnodeEps->eps[0].ep));
+  if (taosArrayGetSize(pMgmt->pDnodeEps) == 0) {
+    SDnodeEp dnodeEp = {0};
+    dnodeEp.isMnode = 1;
+    taosGetFqdnPortFromEp(pDnode->cfg.firstEp, &dnodeEp.ep);
+    taosArrayPush(pMgmt->pDnodeEps, &dnodeEp);
   }
 
-  dndResetDnodes(pDnode, pMgmt->dnodeEps);
+  dndResetDnodes(pDnode, pMgmt->pDnodeEps);
 
   terrno = 0;
   return 0;
@@ -329,13 +325,15 @@ static int32_t dndWriteDnodes(SDnode *pDnode) {
   len += snprintf(content + len, maxLen - len, "  \"clusterId\": \"%" PRId64 "\",\n", pMgmt->clusterId);
   len += snprintf(content + len, maxLen - len, "  \"dropped\": %d,\n", pMgmt->dropped);
   len += snprintf(content + len, maxLen - len, "  \"dnodes\": [{\n");
-  for (int32_t i = 0; i < pMgmt->dnodeEps->num; ++i) {
-    SDnodeEp *pDnodeEp = &pMgmt->dnodeEps->eps[i];
+
+  int32_t numOfEps = (int32_t)taosArrayGetSize(pMgmt->pDnodeEps);
+  for (int32_t i = 0; i < numOfEps; ++i) {
+    SDnodeEp *pDnodeEp = taosArrayGet(pMgmt->pDnodeEps, i);
     len += snprintf(content + len, maxLen - len, "    \"id\": %d,\n", pDnodeEp->id);
     len += snprintf(content + len, maxLen - len, "    \"fqdn\": \"%s\",\n", pDnodeEp->ep.fqdn);
     len += snprintf(content + len, maxLen - len, "    \"port\": %u,\n", pDnodeEp->ep.port);
     len += snprintf(content + len, maxLen - len, "    \"isMnode\": %d\n", pDnodeEp->isMnode);
-    if (i < pMgmt->dnodeEps->num - 1) {
+    if (i < numOfEps - 1) {
       len += snprintf(content + len, maxLen - len, "  },{\n");
     } else {
       len += snprintf(content + len, maxLen - len, "  }]\n");
@@ -355,40 +353,39 @@ static int32_t dndWriteDnodes(SDnode *pDnode) {
 }
 
 void dndSendStatusReq(SDnode *pDnode) {
-  int32_t contLen = sizeof(SStatusReq) + TSDB_MAX_VNODES * sizeof(SVnodeLoad);
-
-  SStatusReq *pStatus = rpcMallocCont(contLen);
-  if (pStatus == NULL) {
-    dError("failed to malloc status message");
-    return;
-  }
+  SStatusReq req = {0};
 
   SDnodeMgmt *pMgmt = &pDnode->dmgmt;
   taosRLockLatch(&pMgmt->latch);
-  pStatus->sver = htonl(pDnode->env.sver);
-  pStatus->dver = htobe64(pMgmt->dver);
-  pStatus->dnodeId = htonl(pMgmt->dnodeId);
-  pStatus->clusterId = htobe64(pMgmt->clusterId);
-  pStatus->rebootTime = htobe64(pMgmt->rebootTime);
-  pStatus->updateTime = htobe64(pMgmt->updateTime);
-  pStatus->numOfCores = htonl(pDnode->env.numOfCores);
-  pStatus->numOfSupportVnodes = htonl(pDnode->cfg.numOfSupportVnodes);
-  tstrncpy(pStatus->dnodeEp, pDnode->cfg.localEp, TSDB_EP_LEN);
-
-  pStatus->clusterCfg.statusInterval = htonl(pDnode->cfg.statusInterval);
-  pStatus->clusterCfg.checkTime = 0;
+  req.sver = pDnode->env.sver;
+  req.dver = pMgmt->dver;
+  req.dnodeId = pMgmt->dnodeId;
+  req.clusterId = pMgmt->clusterId;
+  req.rebootTime = pMgmt->rebootTime;
+  req.updateTime = pMgmt->updateTime;
+  req.numOfCores = pDnode->env.numOfCores;
+  req.numOfSupportVnodes = pDnode->cfg.numOfSupportVnodes;
+  memcpy(req.dnodeEp, pDnode->cfg.localEp, TSDB_EP_LEN);
+
+  req.clusterCfg.statusInterval = pDnode->cfg.statusInterval;
+  req.clusterCfg.checkTime = 0;
   char timestr[32] = "1970-01-01 00:00:00.00";
-  (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
-  pStatus->clusterCfg.checkTime = htonl(pStatus->clusterCfg.checkTime);
-  tstrncpy(pStatus->clusterCfg.timezone, pDnode->env.timezone, TSDB_TIMEZONE_LEN);
-  tstrncpy(pStatus->clusterCfg.locale, pDnode->env.locale, TSDB_LOCALE_LEN);
-  tstrncpy(pStatus->clusterCfg.charset, pDnode->env.charset, TSDB_LOCALE_LEN);
+  (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
+  memcpy(req.clusterCfg.timezone, pDnode->env.timezone, TSDB_TIMEZONE_LEN);
+  memcpy(req.clusterCfg.locale, pDnode->env.locale, TSDB_LOCALE_LEN);
+  memcpy(req.clusterCfg.charset, pDnode->env.charset, TSDB_LOCALE_LEN);
   taosRUnLockLatch(&pMgmt->latch);
 
-  dndGetVnodeLoads(pDnode, &pStatus->vnodeLoads);
-  contLen = sizeof(SStatusReq) + pStatus->vnodeLoads.num * sizeof(SVnodeLoad);
+  req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad));
+  dndGetVnodeLoads(pDnode, req.pVloads);
 
-  SRpcMsg rpcMsg = {.pCont = pStatus, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)9527};
+  int32_t contLen = tSerializeSStatusReq(NULL, &req);
+  void   *pHead = rpcMallocCont(contLen);
+  void   *pBuf = pHead;
+  tSerializeSStatusReq(&pBuf, &req);
+  taosArrayDestroy(req.pVloads);
+
+  SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)9527};
   pMgmt->statusSent = 1;
 
   dTrace("pDnode:%p, send status req to mnode", pDnode);
@@ -407,18 +404,20 @@ static void dndUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) {
   }
 }
 
-static void dndUpdateDnodeEps(SDnode *pDnode, SDnodeEps *pDnodeEps) {
-  if (pDnodeEps == NULL || pDnodeEps->num <= 0) return;
+static void dndUpdateDnodeEps(SDnode *pDnode, SArray *pDnodeEps) {
+  int32_t numOfEps = taosArrayGetSize(pDnodeEps);
+  if (numOfEps <= 0) return;
 
   SDnodeMgmt *pMgmt = &pDnode->dmgmt;
   taosWLockLatch(&pMgmt->latch);
 
-  if (pDnodeEps->num != pMgmt->dnodeEps->num) {
+  int32_t numOfEpsOld = (int32_t)taosArrayGetSize(pMgmt->pDnodeEps);
+  if (numOfEps != numOfEpsOld) {
     dndResetDnodes(pDnode, pDnodeEps);
     dndWriteDnodes(pDnode);
   } else {
-    int32_t size = pDnodeEps->num * sizeof(SDnodeEp) + sizeof(SDnodeEps);
-    if (memcmp(pMgmt->dnodeEps, pDnodeEps, size) != 0) {
+    int32_t size = numOfEps * sizeof(SDnodeEp);
+    if (memcmp(pMgmt->pDnodeEps->pData, pDnodeEps->pData, size) != 0) {
       dndResetDnodes(pDnode, pDnodeEps);
       dndWriteDnodes(pDnode);
     }
@@ -431,33 +430,21 @@ static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pRsp) {
   SDnodeMgmt *pMgmt = &pDnode->dmgmt;
 
   if (pRsp->code != TSDB_CODE_SUCCESS) {
-    pMgmt->statusSent = 0;
     if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pMgmt->dropped && pMgmt->dnodeId > 0) {
       dInfo("dnode:%d, set to dropped since not exist in mnode", pMgmt->dnodeId);
       pMgmt->dropped = 1;
       dndWriteDnodes(pDnode);
     }
-    return;
-  }
-
-  if (pRsp->pCont != NULL && pRsp->contLen != 0) {
-    SStatusRsp *pStatus = pRsp->pCont;
-    pMgmt->dver = htobe64(pStatus->dver);
-
-    SDnodeCfg *pCfg = &pStatus->dnodeCfg;
-    pCfg->dnodeId = htonl(pCfg->dnodeId);
-    pCfg->clusterId = htobe64(pCfg->clusterId);
-    dndUpdateDnodeCfg(pDnode, pCfg);
-
-    SDnodeEps *pDnodeEps = &pStatus->dnodeEps;
-    pDnodeEps->num = htonl(pDnodeEps->num);
-    for (int32_t i = 0; i < pDnodeEps->num; ++i) {
-      pDnodeEps->eps[i].id = htonl(pDnodeEps->eps[i].id);
-      pDnodeEps->eps[i].ep.port = htons(pDnodeEps->eps[i].ep.port);
+  } else {
+    SStatusRsp statusRsp = {0};
+    if (pRsp->pCont != NULL && pRsp->contLen != 0 && tDeserializeSStatusRsp(pRsp->pCont, &statusRsp) != NULL) {
+      pMgmt->dver = statusRsp.dver;
+      dndUpdateDnodeCfg(pDnode, &statusRsp.dnodeCfg);
+      dndUpdateDnodeEps(pDnode, statusRsp.pDnodeEps);
     }
-
-    dndUpdateDnodeEps(pDnode, pDnodeEps);
+    taosArrayDestroy(statusRsp.pDnodeEps);
   }
+
   pMgmt->statusSent = 0;
 }
 
@@ -572,9 +559,9 @@ void dndCleanupMgmt(SDnode *pDnode) {
   SDnodeMgmt *pMgmt = &pDnode->dmgmt;
   taosWLockLatch(&pMgmt->latch);
 
-  if (pMgmt->dnodeEps != NULL) {
-    free(pMgmt->dnodeEps);
-    pMgmt->dnodeEps = NULL;
+  if (pMgmt->pDnodeEps != NULL) {
+    taosArrayDestroy(pMgmt->pDnodeEps);
+    pMgmt->pDnodeEps = NULL;
   }
 
   if (pMgmt->dnodeHash != NULL) {

source/dnode/mgmt/impl/src/dndMnode.c

@@ -256,6 +256,12 @@ static bool dndNeedDeployMnode(SDnode *pDnode) {
 
 static int32_t dndPutMsgToMWriteQ(SDnode *pDnode, SRpcMsg *pRpcMsg) {
   dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.writeWorker, pRpcMsg);
+  return 0;
+}
+
+static int32_t dndPutMsgToMReadQ(SDnode *pDnode, SRpcMsg* pRpcMsg) {
+  dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.readWorker, pRpcMsg);
+  return 0;
 }
 
 static void dndInitMnodeOption(SDnode *pDnode, SMnodeOpt *pOption) {
@@ -264,6 +270,7 @@ static void dndInitMnodeOption(SDnode *pDnode, SMnodeOpt *pOption) {
   pOption->sendReqToMnodeFp = dndSendReqToMnode;
   pOption->sendRedirectRspFp = dndSendRedirectRsp;
   pOption->putReqToMWriteQFp = dndPutMsgToMWriteQ;
+  pOption->putReqToMReadQFp = dndPutMsgToMReadQ;
   pOption->dnodeId = dndGetDnodeId(pDnode);
   pOption->clusterId = dndGetClusterId(pDnode);
   pOption->cfg.sver = pDnode->env.sver;

source/dnode/mgmt/impl/src/dndVnodes.c

@@ -1008,27 +1008,21 @@ void dndCleanupVnodes(SDnode *pDnode) {
   dInfo("dnode-vnodes is cleaned up");
 }
 
-void dndGetVnodeLoads(SDnode *pDnode, SVnodeLoads *pLoads) {
+void dndGetVnodeLoads(SDnode *pDnode, SArray *pLoads) {
   SVnodesMgmt *pMgmt = &pDnode->vmgmt;
 
   taosRLockLatch(&pMgmt->latch);
-  pLoads->num = taosHashGetSize(pMgmt->hash);
 
   int32_t v = 0;
-  void *  pIter = taosHashIterate(pMgmt->hash, NULL);
+  void   *pIter = taosHashIterate(pMgmt->hash, NULL);
   while (pIter) {
     SVnodeObj **ppVnode = pIter;
     if (ppVnode == NULL || *ppVnode == NULL) continue;
 
-    SVnodeObj * pVnode = *ppVnode;
-    SVnodeLoad *pLoad = &pLoads->data[v++];
-
-    vnodeGetLoad(pVnode->pImpl, pLoad);
-    pLoad->vgId = htonl(pLoad->vgId);
-    pLoad->totalStorage = htobe64(pLoad->totalStorage);
-    pLoad->compStorage = htobe64(pLoad->compStorage);
-    pLoad->pointsWritten = htobe64(pLoad->pointsWritten);
-    pLoad->tablesNum = htobe64(pLoad->tablesNum);
+    SVnodeObj *pVnode = *ppVnode;
+    SVnodeLoad vload = {0};
+    vnodeGetLoad(pVnode->pImpl, &vload);
+    taosArrayPush(pLoads, &vload);
 
     pIter = taosHashIterate(pMgmt->hash, pIter);
   }

source/dnode/mgmt/impl/test/vnode/vnode.cpp

@@ -108,7 +108,7 @@ TEST_F(DndTestVnode, 01_Create_Vnode) {
   }
 }
 
-TEST_F(DndTestVnode, 02_ALTER_Vnode) {
+TEST_F(DndTestVnode, 02_Alter_Vnode) {
   for (int i = 0; i < 3; ++i) {
     int32_t contLen = sizeof(SAlterVnodeReq);
 
@@ -219,7 +219,7 @@ TEST_F(DndTestVnode, 03_Create_Stb) {
   }
 }
 
-TEST_F(DndTestVnode, 04_ALTER_Stb) {
+TEST_F(DndTestVnode, 04_Alter_Stb) {
   for (int i = 0; i < 1; ++i) {
     SVCreateTbReq req = {0};
     req.ver = 0;
@@ -310,7 +310,7 @@ TEST_F(DndTestVnode, 05_DROP_Stb) {
   }
 }
 
-TEST_F(DndTestVnode, 06_DROP_Vnode) {
+TEST_F(DndTestVnode, 06_Drop_Vnode) {
   for (int i = 0; i < 3; ++i) {
     int32_t contLen = sizeof(SDropVnodeReq);
 

source/dnode/mnode/impl/inc/mndConsumer.h

@@ -28,6 +28,8 @@ void    mndCleanupConsumer(SMnode *pMnode);
 SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int64_t consumerId);
 void            mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer);
 
+SMqConsumerObj* mndCreateConsumer(int64_t consumerId, const char* cgroup);
+
 SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer);
 SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw);
 

source/dnode/mnode/impl/inc/mndDef.h

@@ -304,9 +304,10 @@ typedef struct {
   int32_t  nextColId;
   int32_t  numOfColumns;
   int32_t  numOfTags;
-  SSchema* pTags;
   SSchema* pColumns;
+  SSchema* pTags;
   SRWLatch lock;
+  char     comment[TSDB_STB_COMMENT_LEN];
 } SStbObj;
 
 typedef struct {
@@ -343,42 +344,22 @@ typedef struct {
   char    payload[];
 } SShowObj;
 
-#if 0
-typedef struct SConsumerObj {
-  uint64_t uid;
-  int64_t  createTime;
-  int64_t  updateTime;
-  //uint64_t dbUid;
-  int32_t  version;
-  SRWLatch lock;
-  SArray*   topics;
-} SConsumerObj;
-
-typedef struct SMqTopicConsumer {
-  int64_t consumerId;
-  SList*  topicList;
-} SMqTopicConsumer;
-#endif
-
-typedef struct SMqConsumerEp {
-  int32_t      vgId;  // -1 for unassigned
-  int32_t      status;
-  SEpSet       epSet;
-  int64_t      consumerId;  // -1 for unassigned
-  int64_t      lastConsumerHbTs;
-  int64_t      lastVgHbTs;
-  char*        qmsg;
+typedef struct {
+  int32_t vgId;  // -1 for unassigned
+  int32_t status;
+  SEpSet  epSet;
+  int64_t oldConsumerId;
+  int64_t consumerId;  // -1 for unassigned
+  char*   qmsg;
 } SMqConsumerEp;
 
-static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) {
+static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, const SMqConsumerEp* pConsumerEp) {
   int32_t tlen = 0;
   tlen += taosEncodeFixedI32(buf, pConsumerEp->vgId);
   tlen += taosEncodeFixedI32(buf, pConsumerEp->status);
   tlen += taosEncodeSEpSet(buf, &pConsumerEp->epSet);
+  tlen += taosEncodeFixedI64(buf, pConsumerEp->oldConsumerId);
   tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId);
-  tlen += taosEncodeFixedI64(buf, pConsumerEp->lastConsumerHbTs);
-  tlen += taosEncodeFixedI64(buf, pConsumerEp->lastVgHbTs);
-  //tlen += tEncodeSSubQueryMsg(buf, &pConsumerEp->qExec);
   tlen += taosEncodeString(buf, pConsumerEp->qmsg);
   return tlen;
 }
@@ -387,10 +368,8 @@ static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsu
   buf = taosDecodeFixedI32(buf, &pConsumerEp->vgId);
   buf = taosDecodeFixedI32(buf, &pConsumerEp->status);
   buf = taosDecodeSEpSet(buf, &pConsumerEp->epSet);
+  buf = taosDecodeFixedI64(buf, &pConsumerEp->oldConsumerId);
   buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId);
-  buf = taosDecodeFixedI64(buf, &pConsumerEp->lastConsumerHbTs);
-  buf = taosDecodeFixedI64(buf, &pConsumerEp->lastVgHbTs);
-  //buf = tDecodeSSubQueryMsg(buf, &pConsumerEp->qExec);
   buf = taosDecodeString(buf, &pConsumerEp->qmsg);
   return buf;
 }
@@ -401,97 +380,89 @@ static FORCE_INLINE void tDeleteSMqConsumerEp(SMqConsumerEp* pConsumerEp) {
   }
 }
 
-// unit for rebalance
-typedef struct SMqSubscribeObj {
-  char    key[TSDB_SUBSCRIBE_KEY_LEN];
-  int32_t epoch;
-  // TODO: replace with priority queue
-  int32_t nextConsumerIdx;
-  SArray* availConsumer;  // SArray<int64_t> (consumerId)
-  SArray* assigned;       // SArray<SMqConsumerEp>
-  SArray* idleConsumer;   // SArray<SMqConsumerEp>
-  SArray* lostConsumer;   // SArray<SMqConsumerEp>
-  SArray* unassignedVg;   // SArray<SMqConsumerEp>
-} SMqSubscribeObj;
+typedef struct {
+  int64_t consumerId;
+  SArray* vgInfo;  // SArray<SMqConsumerEp>
+} SMqSubConsumer;
 
-static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() {
-  SMqSubscribeObj* pSub = malloc(sizeof(SMqSubscribeObj));
-  if (pSub == NULL) {
-    return NULL;
+static FORCE_INLINE int32_t tEncodeSMqSubConsumer(void** buf, const SMqSubConsumer* pConsumer) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI64(buf, pConsumer->consumerId);
+  int32_t sz = taosArrayGetSize(pConsumer->vgInfo);
+  for (int32_t i = 0; i < sz; i++) {
+    SMqConsumerEp* pCEp = taosArrayGet(pConsumer->vgInfo, i);
+    tlen += tEncodeSMqConsumerEp(buf, pCEp);
   }
-  pSub->key[0] = 0;
-  pSub->epoch = 0;
+  return tlen;
+}
 
-  pSub->availConsumer = taosArrayInit(0, sizeof(int64_t));
-  if (pSub->availConsumer == NULL) {
-    free(pSub);
-    return NULL;
+static FORCE_INLINE void* tDecodeSMqSubConsumer(void** buf, SMqSubConsumer* pConsumer) {
+  int32_t sz;
+  buf = taosDecodeFixedI64(buf, &pConsumer->consumerId);
+  buf = taosDecodeFixedI32(buf, &sz);
+  pConsumer->vgInfo = taosArrayInit(sz, sizeof(SMqConsumerEp));
+  for (int32_t i = 0; i < sz; i++) {
+    SMqConsumerEp consumerEp;
+    buf = tDecodeSMqConsumerEp(buf, &consumerEp);
+    taosArrayPush(pConsumer->vgInfo, &consumerEp);
   }
-  pSub->assigned = taosArrayInit(0, sizeof(SMqConsumerEp));
-  if (pSub->assigned == NULL) {
-    taosArrayDestroy(pSub->availConsumer);
-    free(pSub);
-    return NULL;
+  return buf;
+}
+
+static FORCE_INLINE void tDeleteSMqSubConsumer(SMqSubConsumer* pSubConsumer) {
+  if (pSubConsumer->vgInfo) {
+    taosArrayDestroyEx(pSubConsumer->vgInfo, (void (*)(void*))tDeleteSMqConsumerEp);
+    pSubConsumer->vgInfo = NULL;
   }
-  pSub->lostConsumer = taosArrayInit(0, sizeof(SMqConsumerEp));
-  if (pSub->lostConsumer == NULL) {
-    taosArrayDestroy(pSub->availConsumer);
-    taosArrayDestroy(pSub->assigned);
-    free(pSub);
+}
+
+typedef struct {
+  char    key[TSDB_SUBSCRIBE_KEY_LEN];
+  int32_t status;
+  int32_t vgNum;
+  SArray* consumers;     // SArray<SMqSubConsumer>
+  SArray* unassignedVg;  // SArray<SMqConsumerEp>
+} SMqSubscribeObj;
+
+static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() {
+  SMqSubscribeObj* pSub = calloc(1, sizeof(SMqSubscribeObj));
+  if (pSub == NULL) {
     return NULL;
   }
-  pSub->idleConsumer = taosArrayInit(0, sizeof(SMqConsumerEp));
-  if (pSub->idleConsumer == NULL) {
-    taosArrayDestroy(pSub->availConsumer);
-    taosArrayDestroy(pSub->assigned);
-    taosArrayDestroy(pSub->lostConsumer);
-    free(pSub);
-    return NULL;
+  pSub->consumers = taosArrayInit(0, sizeof(SMqSubConsumer));
+  if (pSub->consumers == NULL) {
+    goto _err;
   }
   pSub->unassignedVg = taosArrayInit(0, sizeof(SMqConsumerEp));
   if (pSub->unassignedVg == NULL) {
-    taosArrayDestroy(pSub->availConsumer);
-    taosArrayDestroy(pSub->assigned);
-    taosArrayDestroy(pSub->lostConsumer);
-    taosArrayDestroy(pSub->idleConsumer);
-    free(pSub);
-    return NULL;
+    goto _err;
   }
+
+  pSub->key[0] = 0;
+  pSub->vgNum = 0;
+  pSub->status = 0;
+
   return pSub;
+
+_err:
+  tfree(pSub->unassignedVg);
+  tfree(pSub->consumers);
+  tfree(pSub);
+  return NULL;
 }
 
 static FORCE_INLINE int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub) {
   int32_t tlen = 0;
   tlen += taosEncodeString(buf, pSub->key);
-  tlen += taosEncodeFixedI32(buf, pSub->epoch);
+  tlen += taosEncodeFixedI32(buf, pSub->vgNum);
+  tlen += taosEncodeFixedI32(buf, pSub->status);
   int32_t sz;
 
-  sz = taosArrayGetSize(pSub->availConsumer);
-  tlen += taosEncodeFixedI32(buf, sz);
-  for (int32_t i = 0; i < sz; i++) {
-    int64_t* pConsumerId = taosArrayGet(pSub->availConsumer, i);
-    tlen += taosEncodeFixedI64(buf, *pConsumerId);
-  }
-
-  sz = taosArrayGetSize(pSub->assigned);
-  tlen += taosEncodeFixedI32(buf, sz);
-  for (int32_t i = 0; i < sz; i++) {
-    SMqConsumerEp* pCEp = taosArrayGet(pSub->assigned, i);
-    tlen += tEncodeSMqConsumerEp(buf, pCEp);
-  }
-
-  sz = taosArrayGetSize(pSub->lostConsumer);
+  sz = taosArrayGetSize(pSub->consumers);
   tlen += taosEncodeFixedI32(buf, sz);
   for (int32_t i = 0; i < sz; i++) {
-    SMqConsumerEp* pCEp = taosArrayGet(pSub->lostConsumer, i);
-    tlen += tEncodeSMqConsumerEp(buf, pCEp);
-  }
-
-  sz = taosArrayGetSize(pSub->idleConsumer);
-  tlen += taosEncodeFixedI32(buf, sz);
-  for (int32_t i = 0; i < sz; i++) {
-    SMqConsumerEp* pCEp = taosArrayGet(pSub->idleConsumer, i);
-    tlen += tEncodeSMqConsumerEp(buf, pCEp);
+    SMqSubConsumer* pSubConsumer = taosArrayGet(pSub->consumers, i);
+    tlen += tEncodeSMqSubConsumer(buf, pSubConsumer);
   }
 
   sz = taosArrayGetSize(pSub->unassignedVg);
@@ -506,68 +477,25 @@ static FORCE_INLINE int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeOb
 
 static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) {
   buf = taosDecodeStringTo(buf, pSub->key);
-  buf = taosDecodeFixedI32(buf, &pSub->epoch);
+  buf = taosDecodeFixedI32(buf, &pSub->vgNum);
+  buf = taosDecodeFixedI32(buf, &pSub->status);
 
   int32_t sz;
 
   buf = taosDecodeFixedI32(buf, &sz);
-  pSub->availConsumer = taosArrayInit(sz, sizeof(int64_t));
-  if (pSub->availConsumer == NULL) {
+  pSub->consumers = taosArrayInit(sz, sizeof(SMqSubConsumer));
+  if (pSub->consumers == NULL) {
     return NULL;
   }
   for (int32_t i = 0; i < sz; i++) {
-    int64_t consumerId;
-    buf = taosDecodeFixedI64(buf, &consumerId);
-    taosArrayPush(pSub->availConsumer, &consumerId);
+    SMqSubConsumer subConsumer = {0};
+    buf = tDecodeSMqSubConsumer(buf, &subConsumer);
+    taosArrayPush(pSub->consumers, &subConsumer);
   }
 
-  buf = taosDecodeFixedI32(buf, &sz);
-  pSub->assigned = taosArrayInit(sz, sizeof(SMqConsumerEp));
-  if (pSub->assigned == NULL) {
-    taosArrayDestroy(pSub->availConsumer);
-    return NULL;
-  }
-  for (int32_t i = 0; i < sz; i++) {
-    SMqConsumerEp cEp = {0};
-    buf = tDecodeSMqConsumerEp(buf, &cEp);
-    taosArrayPush(pSub->assigned, &cEp);
-  }
-
-  buf = taosDecodeFixedI32(buf, &sz);
-  pSub->lostConsumer = taosArrayInit(sz, sizeof(SMqConsumerEp));
-  if (pSub->lostConsumer == NULL) {
-    taosArrayDestroy(pSub->availConsumer);
-    taosArrayDestroy(pSub->assigned);
-    return NULL;
-  }
-  for (int32_t i = 0; i < sz; i++) {
-    SMqConsumerEp cEp = {0};
-    buf = tDecodeSMqConsumerEp(buf, &cEp);
-    taosArrayPush(pSub->lostConsumer, &cEp);
-  }
-
-  buf = taosDecodeFixedI32(buf, &sz);
-  pSub->idleConsumer = taosArrayInit(sz, sizeof(SMqConsumerEp));
-  if (pSub->idleConsumer == NULL) {
-    taosArrayDestroy(pSub->availConsumer);
-    taosArrayDestroy(pSub->assigned);
-    taosArrayDestroy(pSub->lostConsumer);
-    return NULL;
-  }
-  for (int32_t i = 0; i < sz; i++) {
-    SMqConsumerEp cEp = {0};
-    buf = tDecodeSMqConsumerEp(buf, &cEp);
-    taosArrayPush(pSub->idleConsumer, &cEp);
-  }
-
-
   buf = taosDecodeFixedI32(buf, &sz);
   pSub->unassignedVg = taosArrayInit(sz, sizeof(SMqConsumerEp));
   if (pSub->unassignedVg == NULL) {
-    taosArrayDestroy(pSub->availConsumer);
-    taosArrayDestroy(pSub->assigned);
-    taosArrayDestroy(pSub->lostConsumer);
-    taosArrayDestroy(pSub->idleConsumer);
     return NULL;
   }
   for (int32_t i = 0; i < sz; i++) {
@@ -575,50 +503,29 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub)
     buf = tDecodeSMqConsumerEp(buf, &cEp);
     taosArrayPush(pSub->unassignedVg, &cEp);
   }
-
   return buf;
 }
 
 static FORCE_INLINE void tDeleteSMqSubscribeObj(SMqSubscribeObj* pSub) {
-  if (pSub->availConsumer) {
-    taosArrayDestroy(pSub->availConsumer);
-    pSub->availConsumer = NULL;
-  }
-  if (pSub->assigned) {
-    //taosArrayDestroyEx(pSub->assigned, (void (*)(void*))tDeleteSMqConsumerEp);
-    taosArrayDestroy(pSub->assigned);
-    pSub->assigned = NULL;
+  if (pSub->consumers) {
+    taosArrayDestroyEx(pSub->consumers, (void (*)(void*))tDeleteSMqSubConsumer);
+    //taosArrayDestroy(pSub->consumers);
+    pSub->consumers = NULL;
   }
+
   if (pSub->unassignedVg) {
-    //taosArrayDestroyEx(pSub->unassignedVg, (void (*)(void*))tDeleteSMqConsumerEp);
-    taosArrayDestroy(pSub->unassignedVg);
+    taosArrayDestroyEx(pSub->unassignedVg, (void (*)(void*))tDeleteSMqConsumerEp);
+    //taosArrayDestroy(pSub->unassignedVg);
     pSub->unassignedVg = NULL;
   }
-  if (pSub->idleConsumer) {
-    //taosArrayDestroyEx(pSub->idleConsumer, (void (*)(void*))tDeleteSMqConsumerEp);
-    taosArrayDestroy(pSub->idleConsumer);
-    pSub->idleConsumer = NULL;
-  }
-  if (pSub->lostConsumer) {
-    //taosArrayDestroyEx(pSub->lostConsumer, (void (*)(void*))tDeleteSMqConsumerEp);
-    taosArrayDestroy(pSub->lostConsumer);
-    pSub->lostConsumer = NULL;
-  }
 }
 
-typedef struct SMqCGroup {
-  char    name[TSDB_CONSUMER_GROUP_LEN];
-  int32_t status;       // 0 - uninitialized, 1 - wait rebalance, 2- normal
-  SList*  consumerIds;  // SList<int64_t>
-  SList*  idleVGroups;  // SList<int32_t>
-} SMqCGroup;
-
-typedef struct SMqTopicObj {
+typedef struct {
   char     name[TSDB_TOPIC_FNAME_LEN];
   char     db[TSDB_DB_FNAME_LEN];
   int64_t  createTime;
   int64_t  updateTime;
-  uint64_t uid;
+  int64_t  uid;
   int64_t  dbUid;
   int32_t  version;
   SRWLatch lock;
@@ -626,79 +533,23 @@ typedef struct SMqTopicObj {
   char*    sql;
   char*    logicalPlan;
   char*    physicalPlan;
-  // SHashObj *cgroups;  // SHashObj<SMqCGroup>
-  // SHashObj *consumers;  // SHashObj<SMqConsumerObj>
 } SMqTopicObj;
 
-// TODO: add cache and change name to id
-typedef struct SMqConsumerTopic {
-  char    name[TSDB_TOPIC_FNAME_LEN];
-  int32_t epoch;
-  // vg assigned to the consumer on the topic
-  SArray* pVgInfo;  // SArray<int32_t>
-} SMqConsumerTopic;
-
-static FORCE_INLINE SMqConsumerTopic* tNewConsumerTopic(int64_t consumerId, SMqTopicObj* pTopic,
-                                                        SMqSubscribeObj* pSub, int64_t* oldConsumerId) {
-  SMqConsumerTopic* pCTopic = malloc(sizeof(SMqConsumerTopic));
-  if (pCTopic == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
-  }
-  strcpy(pCTopic->name, pTopic->name);
-  pCTopic->epoch = 0;
-  pCTopic->pVgInfo = taosArrayInit(0, sizeof(int32_t));
-
-  int32_t unassignedVgSz = taosArrayGetSize(pSub->unassignedVg);
-  if (unassignedVgSz > 0) {
-    SMqConsumerEp* pCEp = taosArrayPop(pSub->unassignedVg);
-    *oldConsumerId = pCEp->consumerId;
-    pCEp->consumerId = consumerId;
-    taosArrayPush(pCTopic->pVgInfo, &pCEp->vgId);
-    taosArrayPush(pSub->assigned, pCEp);
-  }
-  return pCTopic;
-}
-
-static FORCE_INLINE int32_t tEncodeSMqConsumerTopic(void** buf, SMqConsumerTopic* pConsumerTopic) {
-  int32_t tlen = 0;
-  tlen += taosEncodeString(buf, pConsumerTopic->name);
-  tlen += taosEncodeFixedI32(buf, pConsumerTopic->epoch);
-  int32_t sz = 0;
-  if (pConsumerTopic->pVgInfo != NULL) {
-    sz = taosArrayGetSize(pConsumerTopic->pVgInfo);
-  }
-  tlen += taosEncodeFixedI32(buf, sz);
-  for (int32_t i = 0; i < sz; i++) {
-    int32_t* pVgInfo = taosArrayGet(pConsumerTopic->pVgInfo, i);
-    tlen += taosEncodeFixedI32(buf, *pVgInfo);
-  }
-  return tlen;
-}
-
-static FORCE_INLINE void* tDecodeSMqConsumerTopic(void* buf, SMqConsumerTopic* pConsumerTopic) {
-  buf = taosDecodeStringTo(buf, pConsumerTopic->name);
-  buf = taosDecodeFixedI32(buf, &pConsumerTopic->epoch);
-  int32_t sz;
-  buf = taosDecodeFixedI32(buf, &sz);
-  pConsumerTopic->pVgInfo = taosArrayInit(sz, sizeof(SMqConsumerTopic));
-  for (int32_t i = 0; i < sz; i++) {
-    int32_t vgInfo;
-    buf = taosDecodeFixedI32(buf, &vgInfo);
-    taosArrayPush(pConsumerTopic->pVgInfo, &vgInfo);
-  }
-  return buf;
-}
-
-typedef struct SMqConsumerObj {
+typedef struct {
   int64_t  consumerId;
   int64_t  connId;
   SRWLatch lock;
   char     cgroup[TSDB_CONSUMER_GROUP_LEN];
-  SArray*  topics;  // SArray<SMqConsumerTopic>
+  SArray*  topics;  // SArray<char*>
   int64_t  epoch;
   // stat
-  int64_t  pollCnt;
+  int64_t pollCnt;
+  // status
+  int32_t status;
+  // heartbeat from the consumer reset hbStatus to 0
+  // each checkConsumerAlive msg add hbStatus by 1
+  // if checkConsumerAlive > CONSUMER_REBALANCE_CNT, mask to lost
+  int32_t hbStatus;
 } SMqConsumerObj;
 
 static FORCE_INLINE int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer) {
@@ -711,88 +562,29 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerO
   int32_t sz = taosArrayGetSize(pConsumer->topics);
   tlen += taosEncodeFixedI32(buf, sz);
   for (int32_t i = 0; i < sz; i++) {
-    SMqConsumerTopic* pConsumerTopic = taosArrayGet(pConsumer->topics, i);
-    tlen += tEncodeSMqConsumerTopic(buf, pConsumerTopic);
+    char* topic = taosArrayGetP(pConsumer->topics, i);
+    tlen += taosEncodeString(buf, topic);
   }
   return tlen;
 }
 
 static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pConsumer) {
+  int32_t sz;
   buf = taosDecodeFixedI64(buf, &pConsumer->consumerId);
   buf = taosDecodeFixedI64(buf, &pConsumer->connId);
   buf = taosDecodeFixedI64(buf, &pConsumer->epoch);
   buf = taosDecodeFixedI64(buf, &pConsumer->pollCnt);
   buf = taosDecodeStringTo(buf, pConsumer->cgroup);
-  int32_t sz;
   buf = taosDecodeFixedI32(buf, &sz);
   pConsumer->topics = taosArrayInit(sz, sizeof(SMqConsumerObj));
   for (int32_t i = 0; i < sz; i++) {
-    SMqConsumerTopic cTopic;
-    buf = tDecodeSMqConsumerTopic(buf, &cTopic);
-    taosArrayPush(pConsumer->topics, &cTopic);
+    char* topic;
+    buf = taosDecodeString(buf, &topic);
+    taosArrayPush(pConsumer->topics, &topic);
   }
   return buf;
 }
 
-typedef struct SMqSubConsumerObj {
-  int64_t consumerUid;  // if -1, unassigned
-  SList*  vgId;         // SList<int32_t>
-} SMqSubConsumerObj;
-
-typedef struct SMqSubCGroupObj {
-  char   name[TSDB_CONSUMER_GROUP_LEN];
-  SList* consumers;  // SList<SMqConsumerObj>
-} SMqSubCGroupObj;
-
-typedef struct SMqSubTopicObj {
-  char     name[TSDB_TOPIC_FNAME_LEN];
-  char     db[TSDB_DB_FNAME_LEN];
-  int64_t  createTime;
-  int64_t  updateTime;
-  int64_t  uid;
-  int64_t  dbUid;
-  int32_t  version;
-  SRWLatch lock;
-  int32_t  sqlLen;
-  char*    sql;
-  char*    logicalPlan;
-  char*    physicalPlan;
-  SList*   cgroups;  // SList<SMqSubCGroupObj>
-} SMqSubTopicObj;
-
-typedef struct SMqConsumerSubObj {
-  int64_t topicUid;
-  SList*  vgIds;  // SList<int64_t>
-} SMqConsumerSubObj;
-
-typedef struct SMqConsumerHbObj {
-  int64_t consumerId;
-  SList*  consumerSubs;  // SList<SMqConsumerSubObj>
-} SMqConsumerHbObj;
-
-typedef struct SMqVGroupSubObj {
-  int64_t topicUid;
-  SList*  consumerIds;  // SList<int64_t>
-} SMqVGroupSubObj;
-
-typedef struct SMqVGroupHbObj {
-  int64_t vgId;
-  SList*  vgSubs;  // SList<SMqVGroupSubObj>
-} SMqVGroupHbObj;
-
-#if 0
-typedef struct SCGroupObj {
-  char     name[TSDB_TOPIC_NAME_LEN];
-  int64_t  createTime;
-  int64_t  updateTime;
-  uint64_t uid;
-  //uint64_t dbUid;
-  int32_t  version;
-  SRWLatch lock;
-  SList*   consumerIds;
-} SCGroupObj;
-#endif
-
 typedef struct SMnodeMsg {
   char    user[TSDB_USER_LEN];
   char    db[TSDB_DB_FNAME_LEN];

source/dnode/mnode/impl/inc/mndInt.h

@@ -96,6 +96,7 @@ typedef struct SMnode {
   SendReqToMnodeFp  sendReqToMnodeFp;
   SendRedirectRspFp sendRedirectRspFp;
   PutReqToMWriteQFp putReqToMWriteQFp;
+  PutReqToMReadQFp  putReqToMReadQFp;
 } SMnode;
 
 int32_t mndSendReqToDnode(SMnode *pMnode, SEpSet *pEpSet, SRpcMsg *rpcMsg);

source/dnode/mnode/impl/inc/mndSubscribe.h

@@ -28,9 +28,6 @@ void    mndCleanupSubscribe(SMnode *pMnode);
 SMqSubscribeObj *mndAcquireSubscribe(SMnode *pMnode, char *CGroup, char *topicName);
 void             mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub);
 
-SSdbRaw *mndSubscribeActionEncode(SMqSubscribeObj *pSub);
-SSdbRow *mndSubscribeActionDecode(SSdbRaw *pRaw);
-
 #ifdef __cplusplus
 }
 #endif

source/dnode/mnode/impl/src/mndConsumer.c

@@ -53,6 +53,19 @@ int32_t mndInitConsumer(SMnode *pMnode) {
 
 void mndCleanupConsumer(SMnode *pMnode) {}
 
+SMqConsumerObj* mndCreateConsumer(int64_t consumerId, const char* cgroup) {
+  SMqConsumerObj* pConsumer = malloc(sizeof(SMqConsumerObj));
+  if (pConsumer == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+  pConsumer->epoch = 1;
+  pConsumer->consumerId = consumerId;
+  strcpy(pConsumer->cgroup, cgroup);
+  taosInitRWLatch(&pConsumer->lock);
+  return pConsumer;
+}
+
 SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) {
   terrno = TSDB_CODE_OUT_OF_MEMORY;
   void* buf = NULL;
@@ -164,148 +177,3 @@ void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer) {
   SSdb *pSdb = pMnode->pSdb;
   sdbRelease(pSdb, pConsumer);
 }
-
-#if 0
-static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg) {
-  SMnode        *pMnode = pMsg->pMnode;
-  STableInfoReq *pInfo = pMsg->rpcMsg.pCont;
-
-  mDebug("consumer:%s, start to retrieve meta", pInfo->tableFname);
-
-  SDbObj *pDb = mndAcquireDbByConsumer(pMnode, pInfo->tableFname);
-  if (pDb == NULL) {
-    terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
-    mError("consumer:%s, failed to retrieve meta since %s", pInfo->tableFname, terrstr());
-    return -1;
-  }
-
-  SConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pInfo->tableFname);
-  if (pConsumer == NULL) {
-    mndReleaseDb(pMnode, pDb);
-    terrno = TSDB_CODE_MND_INVALID_CONSUMER;
-    mError("consumer:%s, failed to get meta since %s", pInfo->tableFname, terrstr());
-    return -1;
-  }
-
-  taosRLockLatch(&pConsumer->lock);
-  int32_t totalCols = pConsumer->numOfColumns + pConsumer->numOfTags;
-  int32_t contLen = sizeof(STableMetaRsp) + totalCols * sizeof(SSchema);
-
-  STableMetaRsp *pMeta = rpcMallocCont(contLen);
-  if (pMeta == NULL) {
-    taosRUnLockLatch(&pConsumer->lock);
-    mndReleaseDb(pMnode, pDb);
-    mndReleaseConsumer(pMnode, pConsumer);
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    mError("consumer:%s, failed to get meta since %s", pInfo->tableFname, terrstr());
-    return -1;
-  }
-
-  memcpy(pMeta->consumerFname, pConsumer->name, TSDB_TABLE_FNAME_LEN);
-  pMeta->numOfTags = htonl(pConsumer->numOfTags);
-  pMeta->numOfColumns = htonl(pConsumer->numOfColumns);
-  pMeta->precision = pDb->cfg.precision;
-  pMeta->tableType = TSDB_SUPER_TABLE;
-  pMeta->update = pDb->cfg.update;
-  pMeta->sversion = htonl(pConsumer->version);
-  pMeta->tuid = htonl(pConsumer->uid);
-
-  for (int32_t i = 0; i < totalCols; ++i) {
-    SSchema *pSchema = &pMeta->pSchema[i];
-    SSchema *pSrcSchema = &pConsumer->pSchema[i];
-    memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN);
-    pSchema->type = pSrcSchema->type;
-    pSchema->colId = htonl(pSrcSchema->colId);
-    pSchema->bytes = htonl(pSrcSchema->bytes);
-  }
-  taosRUnLockLatch(&pConsumer->lock);
-  mndReleaseDb(pMnode, pDb);
-  mndReleaseConsumer(pMnode, pConsumer);
-
-  pMsg->pCont = pMeta;
-  pMsg->contLen = contLen;
-
-  mDebug("consumer:%s, meta is retrieved, cols:%d tags:%d", pInfo->tableFname, pConsumer->numOfColumns, pConsumer->numOfTags);
-  return 0;
-}
-
-static int32_t mndGetNumOfConsumers(SMnode *pMnode, char *dbName, int32_t *pNumOfConsumers) {
-  SSdb *pSdb = pMnode->pSdb;
-
-  SDbObj *pDb = mndAcquireDb(pMnode, dbName);
-  if (pDb == NULL) {
-    terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
-    return -1;
-  }
-
-  int32_t numOfConsumers = 0;
-  void   *pIter = NULL;
-  while (1) {
-    SMqConsumerObj *pConsumer = NULL;
-    pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer);
-    if (pIter == NULL) break;
-
-    numOfConsumers++;
-
-    sdbRelease(pSdb, pConsumer);
-  }
-
-  *pNumOfConsumers = numOfConsumers;
-  return 0;
-}
-
-static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) {
-  SMnode *pMnode = pMsg->pMnode;
-  SSdb   *pSdb = pMnode->pSdb;
-
-  if (mndGetNumOfConsumers(pMnode, pShow->db, &pShow->numOfRows) != 0) {
-    return -1;
-  }
-
-  int32_t  cols = 0;
-  SSchema *pSchema = pMeta->pSchema;
-
-  pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
-  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-  strcpy(pSchema[cols].name, "name");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 8;
-  pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
-  strcpy(pSchema[cols].name, "create_time");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_INT;
-  strcpy(pSchema[cols].name, "columns");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
-  cols++;
-
-  pShow->bytes[cols] = 4;
-  pSchema[cols].type = TSDB_DATA_TYPE_INT;
-  strcpy(pSchema[cols].name, "tags");
-  pSchema[cols].bytes = htonl(pShow->bytes[cols]);
-  cols++;
-
-  pMeta->numOfColumns = htonl(cols);
-  pShow->numOfColumns = cols;
-
-  pShow->offset[0] = 0;
-  for (int32_t i = 1; i < cols; ++i) {
-    pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
-  }
-
-  pShow->numOfRows = sdbGetSize(pSdb, SDB_CONSUMER);
-  pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
-  strcpy(pMeta->tbFname, mndShowStr(pShow->type));
-
-  return 0;
-}
-
-static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) {
-  SSdb *pSdb = pMnode->pSdb;
-  sdbCancelFetch(pSdb, pIter);
-}
-#endif

source/dnode/mnode/impl/src/mndDnode.c

@@ -244,7 +244,7 @@ bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) {
   return true;
 }
 
-static void mndGetDnodeData(SMnode *pMnode, SDnodeEps *pEps, int32_t maxEps) {
+static void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeEps) {
   SSdb *pSdb = pMnode->pSdb;
 
   int32_t numOfEps = 0;
@@ -253,25 +253,20 @@ static void mndGetDnodeData(SMnode *pMnode, SDnodeEps *pEps, int32_t maxEps) {
     SDnodeObj *pDnode = NULL;
     pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode);
     if (pIter == NULL) break;
-    if (numOfEps >= maxEps) {
-      sdbCancelFetch(pSdb, pIter);
-      sdbRelease(pSdb, pDnode);
-      break;
-    }
 
-    SDnodeEp *pEp = &pEps->eps[numOfEps];
-    pEp->id = htonl(pDnode->id);
-    pEp->ep.port = htons(pDnode->port);
-    memcpy(pEp->ep.fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
-    pEp->isMnode = 0;
+    SDnodeEp dnodeEp = {0};
+    dnodeEp.id = pDnode->id;
+    dnodeEp.isMnode = 0;
+    dnodeEp.ep.port = pDnode->port;
+    memcpy(dnodeEp.ep.fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
+
     if (mndIsMnode(pMnode, pDnode->id)) {
-      pEp->isMnode = 1;
+      dnodeEp.isMnode = 1;
     }
-    numOfEps++;
+
     sdbRelease(pSdb, pDnode);
+    taosArrayPush(pDnodeEps, &dnodeEp);
   }
-
-  pEps->num = htonl(numOfEps);
 }
 
 static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) {
@@ -299,50 +294,29 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) {
   return 0;
 }
 
-static void mndParseStatusMsg(SStatusReq *pStatus) {
-  pStatus->sver = htonl(pStatus->sver);
-  pStatus->dver = htobe64(pStatus->dver);
-  pStatus->dnodeId = htonl(pStatus->dnodeId);
-  pStatus->clusterId = htobe64(pStatus->clusterId);
-  pStatus->rebootTime = htobe64(pStatus->rebootTime);
-  pStatus->updateTime = htobe64(pStatus->updateTime);
-  pStatus->numOfCores = htonl(pStatus->numOfCores);
-  pStatus->numOfSupportVnodes = htonl(pStatus->numOfSupportVnodes);
-  pStatus->clusterCfg.statusInterval = htonl(pStatus->clusterCfg.statusInterval);
-  pStatus->clusterCfg.checkTime = htobe64(pStatus->clusterCfg.checkTime);
-  for (int32_t v = 0; v < pStatus->vnodeLoads.num; ++v) {
-    SVnodeLoad *pVload = &pStatus->vnodeLoads.data[v];
-    pVload->vgId = htonl(pVload->vgId);
-    pVload->totalStorage = htobe64(pVload->totalStorage);
-    pVload->compStorage = htobe64(pVload->compStorage);
-    pVload->pointsWritten = htobe64(pVload->pointsWritten);
-    pVload->tablesNum = htobe64(pVload->tablesNum);
-  }
-}
-
 static int32_t mndProcessStatusReq(SMnodeMsg *pReq) {
-  SMnode     *pMnode = pReq->pMnode;
-  SStatusReq *pStatus = pReq->rpcMsg.pCont;
-  SDnodeObj  *pDnode = NULL;
-  int32_t     code = -1;
+  SMnode    *pMnode = pReq->pMnode;
+  SStatusReq statusReq = {0};
+  SDnodeObj *pDnode = NULL;
+  int32_t    code = -1;
 
-  mndParseStatusMsg(pStatus);
+  if (tDeserializeSStatusReq(pReq->rpcMsg.pCont, &statusReq) == NULL) goto PROCESS_STATUS_MSG_OVER;
 
-  if (pStatus->dnodeId == 0) {
-    pDnode = mndAcquireDnodeByEp(pMnode, pStatus->dnodeEp);
+  if (statusReq.dnodeId == 0) {
+    pDnode = mndAcquireDnodeByEp(pMnode, statusReq.dnodeEp);
     if (pDnode == NULL) {
-      mDebug("dnode:%s, not created yet", pStatus->dnodeEp);
+      mDebug("dnode:%s, not created yet", statusReq.dnodeEp);
       terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
       goto PROCESS_STATUS_MSG_OVER;
     }
   } else {
-    pDnode = mndAcquireDnode(pMnode, pStatus->dnodeId);
+    pDnode = mndAcquireDnode(pMnode, statusReq.dnodeId);
     if (pDnode == NULL) {
-      pDnode = mndAcquireDnodeByEp(pMnode, pStatus->dnodeEp);
+      pDnode = mndAcquireDnodeByEp(pMnode, statusReq.dnodeEp);
       if (pDnode != NULL) {
         pDnode->offlineReason = DND_REASON_DNODE_ID_NOT_MATCH;
       }
-      mError("dnode:%d, %s not exist", pStatus->dnodeId, pStatus->dnodeEp);
+      mError("dnode:%d, %s not exist", statusReq.dnodeId, statusReq.dnodeEp);
       terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
       goto PROCESS_STATUS_MSG_OVER;
     }
@@ -350,28 +324,28 @@ static int32_t mndProcessStatusReq(SMnodeMsg *pReq) {
 
   int64_t curMs = taosGetTimestampMs();
   bool    online = mndIsDnodeOnline(pMnode, pDnode, curMs);
-  bool    dnodeChanged = (pStatus->dver != sdbGetTableVer(pMnode->pSdb, SDB_DNODE));
-  bool    reboot = (pDnode->rebootTime != pStatus->rebootTime);
+  bool    dnodeChanged = (statusReq.dver != sdbGetTableVer(pMnode->pSdb, SDB_DNODE));
+  bool    reboot = (pDnode->rebootTime != statusReq.rebootTime);
   bool    needCheck = !online || dnodeChanged || reboot;
 
   if (needCheck) {
-    if (pStatus->sver != pMnode->cfg.sver) {
+    if (statusReq.sver != pMnode->cfg.sver) {
       if (pDnode != NULL) {
         pDnode->offlineReason = DND_REASON_VERSION_NOT_MATCH;
       }
-      mError("dnode:%d, status msg version:%d not match cluster:%d", pStatus->dnodeId, pStatus->sver, pMnode->cfg.sver);
+      mError("dnode:%d, status msg version:%d not match cluster:%d", statusReq.dnodeId, statusReq.sver, pMnode->cfg.sver);
       terrno = TSDB_CODE_MND_INVALID_MSG_VERSION;
       goto PROCESS_STATUS_MSG_OVER;
     }
 
-    if (pStatus->dnodeId == 0) {
+    if (statusReq.dnodeId == 0) {
       mDebug("dnode:%d, %s first access, set clusterId %" PRId64, pDnode->id, pDnode->ep, pMnode->clusterId);
     } else {
-      if (pStatus->clusterId != pMnode->clusterId) {
+      if (statusReq.clusterId != pMnode->clusterId) {
         if (pDnode != NULL) {
           pDnode->offlineReason = DND_REASON_CLUSTER_ID_NOT_MATCH;
         }
-        mError("dnode:%d, clusterId %" PRId64 " not match exist %" PRId64, pDnode->id, pStatus->clusterId,
+        mError("dnode:%d, clusterId %" PRId64 " not match exist %" PRId64, pDnode->id, statusReq.clusterId,
                pMnode->clusterId);
         terrno = TSDB_CODE_MND_INVALID_CLUSTER_ID;
         goto PROCESS_STATUS_MSG_OVER;
@@ -382,7 +356,7 @@ static int32_t mndProcessStatusReq(SMnodeMsg *pReq) {
     }
 
     // Verify whether the cluster parameters are consistent when status change from offline to ready
-    int32_t ret = mndCheckClusterCfgPara(pMnode, &pStatus->clusterCfg);
+    int32_t ret = mndCheckClusterCfgPara(pMnode, &statusReq.clusterCfg);
     if (0 != ret) {
       pDnode->offlineReason = ret;
       mError("dnode:%d, cluster cfg inconsistent since:%s", pDnode->id, offlineReason[ret]);
@@ -396,25 +370,30 @@ static int32_t mndProcessStatusReq(SMnodeMsg *pReq) {
       mDebug("dnode:%d, send dnode eps", pDnode->id);
     }
 
-    pDnode->rebootTime = pStatus->rebootTime;
-    pDnode->numOfCores = pStatus->numOfCores;
-    pDnode->numOfSupportVnodes = pStatus->numOfSupportVnodes;
+    pDnode->rebootTime = statusReq.rebootTime;
+    pDnode->numOfCores = statusReq.numOfCores;
+    pDnode->numOfSupportVnodes = statusReq.numOfSupportVnodes;
 
-    int32_t     numOfEps = mndGetDnodeSize(pMnode);
-    int32_t     contLen = sizeof(SStatusRsp) + numOfEps * sizeof(SDnodeEp);
-    SStatusRsp *pRsp = rpcMallocCont(contLen);
-    if (pRsp == NULL) {
+    SStatusRsp statusRsp = {0};
+    statusRsp.dver = sdbGetTableVer(pMnode->pSdb, SDB_DNODE);
+    statusRsp.dnodeCfg.dnodeId = pDnode->id;
+    statusRsp.dnodeCfg.clusterId = pMnode->clusterId;
+    statusRsp.pDnodeEps = taosArrayInit(mndGetDnodeSize(pMnode), sizeof(SDnodeEp));
+    if (statusRsp.pDnodeEps == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       goto PROCESS_STATUS_MSG_OVER;
     }
 
-    pRsp->dver = htobe64(sdbGetTableVer(pMnode->pSdb, SDB_DNODE));
-    pRsp->dnodeCfg.dnodeId = htonl(pDnode->id);
-    pRsp->dnodeCfg.clusterId = htobe64(pMnode->clusterId);
-    mndGetDnodeData(pMnode, &pRsp->dnodeEps, numOfEps);
+    mndGetDnodeData(pMnode, statusRsp.pDnodeEps);
+
+    int32_t contLen = tSerializeSStatusRsp(NULL, &statusRsp);
+    void   *pHead = rpcMallocCont(contLen);
+    void   *pBuf = pHead;
+    tSerializeSStatusRsp(&pBuf, &statusRsp);
+    taosArrayDestroy(statusRsp.pDnodeEps);
 
     pReq->contLen = contLen;
-    pReq->pCont = pRsp;
+    pReq->pCont = pHead;
   }
 
   pDnode->lastAccessTime = curMs;
@@ -422,6 +401,7 @@ static int32_t mndProcessStatusReq(SMnodeMsg *pReq) {
 
 PROCESS_STATUS_MSG_OVER:
   mndReleaseDnode(pMnode, pDnode);
+  taosArrayDestroy(statusReq.pVloads);
   return code;
 }
 

source/dnode/mnode/impl/src/mnode.c

@@ -76,7 +76,7 @@ static void mndCalMqRebalance(void *param, void *tmrId) {
   if (mndIsMaster(pMnode)) {
     SMqTmrMsg *pMsg = rpcMallocCont(sizeof(SMqTmrMsg));
     SRpcMsg    rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pMsg, .contLen = sizeof(SMqTmrMsg)};
-    pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg);
+    pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg);
   }
 
   taosTmrReset(mndCalMqRebalance, 3000, pMnode, pMnode->timer, &pMnode->mqTimer);
@@ -249,6 +249,7 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) {
   memcpy(&pMnode->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA);
   pMnode->pDnode = pOption->pDnode;
   pMnode->putReqToMWriteQFp = pOption->putReqToMWriteQFp;
+  pMnode->putReqToMReadQFp = pOption->putReqToMReadQFp;
   pMnode->sendReqToDnodeFp = pOption->sendReqToDnodeFp;
   pMnode->sendReqToMnodeFp = pOption->sendReqToMnodeFp;
   pMnode->sendRedirectRspFp = pOption->sendRedirectRspFp;

source/dnode/vnode/src/inc/tsdbReadImpl.h

@@ -143,7 +143,7 @@ int   tsdbLoadBlockIdx(SReadH *pReadh);
 int   tsdbSetReadTable(SReadH *pReadh, STable *pTable);
 int   tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget);
 int   tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo);
-int   tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, int16_t *colIds, int numOfColsIds);
+int   tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, int numOfColsIds);
 int   tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock);
 int   tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx);
 void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx);

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -1638,4 +1638,4 @@ static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *p
 //   }
 
 //   return 0;
-// }
+// }
\ No newline at end of file

source/dnode/vnode/src/tsdb/tsdbReadImpl.c

@@ -272,7 +272,7 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) {
   return 0;
 }
 
-int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, int16_t *colIds, int numOfColsIds) {
+int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, int numOfColsIds) {
   ASSERT(pBlock->numOfSubBlocks > 0);
   int8_t update = pReadh->pRepo->config.update;
 
@@ -580,7 +580,7 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32
   return 0;
 }
 
-static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, int16_t *colIds,
+static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, const int16_t *colIds,
                                      int numOfColIds) {
   ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1);
   ASSERT(colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID);

source/dnode/vnode/src/vnd/vnodeBufferPool.c

@@ -188,4 +188,4 @@ static void vBufPoolDestroyMA(SMemAllocatorFactory *pMAF, SMemAllocator *pMA) {
     vmaReset(pVMA);
     TD_DLIST_APPEND(&(pVnode->pBufPool->free), pVMA);
   }
-}
+}
\ No newline at end of file

source/libs/catalog/inc/catalogInt.h

@@ -48,18 +48,22 @@ enum {
 };
 
 typedef struct SCtgDebug {
-  int32_t lockDebug;
+  bool     lockDebug;
+  bool     cacheDebug;
+  uint32_t showCachePeriodSec;
 } SCtgDebug;
 
 
 typedef struct SCtgTbMetaCache {
   SRWLatch  stbLock;
-  SHashObj *cache;           //key:tbname, value:STableMeta
+  SRWLatch  metaLock;        // RC between cache destroy and all other operations
+  SHashObj *metaCache;       //key:tbname, value:STableMeta
   SHashObj *stbCache;        //key:suid, value:STableMeta*
 } SCtgTbMetaCache;
 
 typedef struct SCtgDBCache {
   SRWLatch         vgLock;
+  uint64_t         dbId;
   int8_t           deleted;
   SDBVgroupInfo   *vgInfo;  
   SCtgTbMetaCache  tbCache;
@@ -81,6 +85,7 @@ typedef struct SCtgRentMgmt {
 
 typedef struct SCatalog {
   uint64_t         clusterId;  
+  SRWLatch         dbLock;
   SHashObj        *dbCache;      //key:dbname, value:SCtgDBCache
   SCtgRentMgmt     dbRent;
   SCtgRentMgmt     stbRent;
@@ -105,6 +110,8 @@ typedef struct SCatalogStat {
 } SCatalogStat;
 
 typedef struct SCatalogMgmt {
+  bool                  exit;
+  SRWLatch              lock;
   SHashObj             *pCluster;     //key: clusterId, value: SCatalog*
   SCatalogStat          stat;
   SCatalogCfg           cfg;
@@ -132,11 +139,8 @@ typedef uint32_t (*tableNameHashFp)(const char *, uint32_t);
 #define ctgDebug(param, ...)  qDebug("CTG:%p " param, pCatalog, __VA_ARGS__)
 #define ctgTrace(param, ...)  qTrace("CTG:%p " param, pCatalog, __VA_ARGS__)
 
-#define CTG_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
-#define CTG_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
-#define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
-
 #define CTG_LOCK_DEBUG(...) do { if (gCTGDebug.lockDebug) { qDebug(__VA_ARGS__); } } while (0)
+#define CTG_CACHE_DEBUG(...) do { if (gCTGDebug.cacheDebug) { qDebug(__VA_ARGS__); } } while (0)
 
 #define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000
 
@@ -172,6 +176,15 @@ typedef uint32_t (*tableNameHashFp)(const char *, uint32_t);
   }                                                       \
 } while (0)
 
+  
+#define CTG_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
+#define CTG_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
+#define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
+
+#define CTG_API_ENTER() do { CTG_LOCK(CTG_READ, &ctgMgmt.lock); if (atomic_load_8(&ctgMgmt.exit)) { CTG_RET(TSDB_CODE_CTG_OUT_OF_SERVICE); }  } while (0)
+#define CTG_API_LEAVE(c) do { int32_t __code = c; CTG_UNLOCK(CTG_READ, &ctgMgmt.lock); CTG_RET(__code); } while (0)
+
+
 
 #ifdef __cplusplus
 }

source/libs/catalog/test/catalogTests.cpp

@@ -39,6 +39,7 @@ namespace {
 extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta,
                                             int32_t *exist);
 extern "C" int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output);
+extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type);
 
 void ctgTestSetPrepareTableMeta();
 void ctgTestSetPrepareCTableMeta();
@@ -49,7 +50,7 @@ bool    ctgTestStop = false;
 bool    ctgTestEnableSleep = false;
 bool    ctgTestDeadLoop = false;
 int32_t ctgTestPrintNum = 200000;
-int32_t ctgTestMTRunSec = 30;
+int32_t ctgTestMTRunSec = 5;
 
 int32_t ctgTestCurrentVgVersion = 0;
 int32_t ctgTestVgVersion = 1;
@@ -107,6 +108,7 @@ void ctgTestInitLogFile() {
   const int32_t maxLogFileNum = 10;
 
   tsAsyncLog = 0;
+  qDebugFlag = 159;
 
   char temp[128] = {0};
   sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix);
@@ -185,7 +187,6 @@ void ctgTestBuildDBVgroup(SDBVgroupInfo **pdbVgroup) {
   ctgTestCurrentVgVersion = dbVgroup->vgVersion;
 
   dbVgroup->hashMethod = 0;
-  dbVgroup->dbId = ctgTestDbId;
   dbVgroup->vgHash = taosHashInit(ctgTestVgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
 
   vgNum = ctgTestGetVgNumFromVgVersion(dbVgroup->vgVersion);
@@ -209,6 +210,45 @@ void ctgTestBuildDBVgroup(SDBVgroupInfo **pdbVgroup) {
   *pdbVgroup = dbVgroup;
 }
 
+
+void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) {
+  strcpy(rspMsg->dbFName, ctgTestDbname);
+  sprintf(rspMsg->tbName, "%s", ctgTestSTablename);
+  sprintf(rspMsg->stbName, "%s", ctgTestSTablename);
+  rspMsg->numOfTags = ctgTestTagNum;
+  rspMsg->numOfColumns = ctgTestColNum;
+  rspMsg->precision = 1 + 1;
+  rspMsg->tableType = TSDB_SUPER_TABLE;
+  rspMsg->update = 1 + 1;
+  rspMsg->sversion = ctgTestSVersion + 1;
+  rspMsg->tversion = ctgTestTVersion + 1;
+  rspMsg->suid = ctgTestSuid + 1;
+  rspMsg->tuid = ctgTestSuid + 1;
+  rspMsg->vgId = 1;
+
+  SSchema *s = NULL;
+  s = &rspMsg->pSchema[0];
+  s->type = TSDB_DATA_TYPE_TIMESTAMP;
+  s->colId = 1;
+  s->bytes = 8;
+  strcpy(s->name, "ts");
+
+  s = &rspMsg->pSchema[1];
+  s->type = TSDB_DATA_TYPE_INT;
+  s->colId = 2;
+  s->bytes = 4;
+  strcpy(s->name, "col1s");
+
+  s = &rspMsg->pSchema[2];
+  s->type = TSDB_DATA_TYPE_BINARY;
+  s->colId = 3;
+  s->bytes = 12 + 1;
+  strcpy(s->name, "tag1s");
+
+  return;
+}
+
+
 void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) {
   SUseDbRsp *rspMsg = NULL;  // todo
 
@@ -592,7 +632,32 @@ void *ctgTestGetDbVgroupThread(void *param) {
   return NULL;
 }
 
-void *ctgTestSetDbVgroupThread(void *param) {
+void *ctgTestSetSameDbVgroupThread(void *param) {
+  struct SCatalog *pCtg = (struct SCatalog *)param;
+  int32_t          code = 0;
+  SDBVgroupInfo    *dbVgroup = NULL;
+  int32_t          n = 0;
+
+  while (!ctgTestStop) {
+    ctgTestBuildDBVgroup(&dbVgroup);
+    code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup);
+    if (code) {
+      assert(0);
+    }
+
+    if (ctgTestEnableSleep) {
+      usleep(rand() % 5);
+    }
+    if (++n % ctgTestPrintNum == 0) {
+      printf("Set:%d\n", n);
+    }
+  }
+
+  return NULL;
+}
+
+
+void *ctgTestSetDiffDbVgroupThread(void *param) {
   struct SCatalog *pCtg = (struct SCatalog *)param;
   int32_t          code = 0;
   SDBVgroupInfo    *dbVgroup = NULL;
@@ -600,7 +665,7 @@ void *ctgTestSetDbVgroupThread(void *param) {
 
   while (!ctgTestStop) {
     ctgTestBuildDBVgroup(&dbVgroup);
-    code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, dbVgroup);
+    code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, ctgTestDbId++, dbVgroup);
     if (code) {
       assert(0);
     }
@@ -616,6 +681,7 @@ void *ctgTestSetDbVgroupThread(void *param) {
   return NULL;
 }
 
+
 void *ctgTestGetCtableMetaThread(void *param) {
   struct SCatalog *pCtg = (struct SCatalog *)param;
   int32_t          code = 0;
@@ -681,6 +747,8 @@ TEST(tableMeta, normalTable) {
   void            *mockPointer = (void *)0x1;
   SVgroupInfo      vgInfo = {0};
 
+  ctgTestInitLogFile();
+
   ctgTestSetPrepareDbVgroups();
 
   initQueryModuleMsgHandle();
@@ -771,6 +839,8 @@ TEST(tableMeta, childTableCase) {
   void            *mockPointer = (void *)0x1;
   SVgroupInfo      vgInfo = {0};
 
+  ctgTestInitLogFile();
+
   ctgTestSetPrepareDbVgroupsAndChildMeta();
 
   initQueryModuleMsgHandle();
@@ -964,6 +1034,124 @@ TEST(tableMeta, superTableCase) {
   catalogDestroy();
 }
 
+TEST(tableMeta, rmStbMeta) {
+  struct SCatalog *pCtg = NULL;
+  void            *mockPointer = (void *)0x1;
+  SVgroupInfo      vgInfo = {0};
+
+  ctgTestInitLogFile();
+
+  ctgTestSetPrepareDbVgroupsAndSuperMeta();
+
+  initQueryModuleMsgHandle();
+
+  int32_t code = catalogInit(NULL);
+  ASSERT_EQ(code, 0);
+
+  // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet);
+  code = catalogGetHandle(ctgTestClusterId, &pCtg);
+  ASSERT_EQ(code, 0);
+
+  SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
+  strcpy(n.dbname, "db1");
+  strcpy(n.tname, ctgTestSTablename);
+
+  STableMeta *tableMeta = NULL;
+  code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta);
+  ASSERT_EQ(code, 0);
+  ASSERT_EQ(tableMeta->vgId, 0);
+  ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE);
+  ASSERT_EQ(tableMeta->sversion, ctgTestSVersion);
+  ASSERT_EQ(tableMeta->tversion, ctgTestTVersion);
+  ASSERT_EQ(tableMeta->uid, ctgTestSuid);
+  ASSERT_EQ(tableMeta->suid, ctgTestSuid);
+  ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum);
+  ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum);
+  ASSERT_EQ(tableMeta->tableInfo.precision, 1);
+  ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
+
+  code = catalogRemoveSTableMeta(pCtg, "1.db1", ctgTestSTablename, ctgTestSuid);
+  ASSERT_EQ(code, 0);
+
+  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1);
+  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 0);
+  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0);
+  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1);
+  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0);
+  
+  catalogDestroy();
+}
+
+TEST(tableMeta, updateStbMeta) {
+  struct SCatalog *pCtg = NULL;
+  void            *mockPointer = (void *)0x1;
+  SVgroupInfo      vgInfo = {0};
+
+  ctgTestInitLogFile();
+
+  ctgTestSetPrepareDbVgroupsAndSuperMeta();
+
+  initQueryModuleMsgHandle();
+
+  int32_t code = catalogInit(NULL);
+  ASSERT_EQ(code, 0);
+
+  // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet);
+  code = catalogGetHandle(ctgTestClusterId, &pCtg);
+  ASSERT_EQ(code, 0);
+
+  SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
+  strcpy(n.dbname, "db1");
+  strcpy(n.tname, ctgTestSTablename);
+
+  STableMeta *tableMeta = NULL;
+  code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta);
+  ASSERT_EQ(code, 0);
+  ASSERT_EQ(tableMeta->vgId, 0);
+  ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE);
+  ASSERT_EQ(tableMeta->sversion, ctgTestSVersion);
+  ASSERT_EQ(tableMeta->tversion, ctgTestTVersion);
+  ASSERT_EQ(tableMeta->uid, ctgTestSuid);
+  ASSERT_EQ(tableMeta->suid, ctgTestSuid);
+  ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum);
+  ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum);
+  ASSERT_EQ(tableMeta->tableInfo.precision, 1);
+  ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
+
+  tfree(tableMeta);
+
+  STableMetaRsp rsp = {0};
+  ctgTestBuildSTableMetaRsp(&rsp);
+
+  code = catalogUpdateSTableMeta(pCtg, &rsp);
+  ASSERT_EQ(code, 0);
+
+  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), 1);
+  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), 1);
+  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 1);
+  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1);
+  ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 1);
+
+  code = catalogGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta);
+  ASSERT_EQ(code, 0);
+  ASSERT_EQ(tableMeta->vgId, 0);
+  ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE);
+  ASSERT_EQ(tableMeta->sversion, ctgTestSVersion + 1);
+  ASSERT_EQ(tableMeta->tversion, ctgTestTVersion + 1);
+  ASSERT_EQ(tableMeta->uid, ctgTestSuid + 1);
+  ASSERT_EQ(tableMeta->suid, ctgTestSuid + 1);
+  ASSERT_EQ(tableMeta->tableInfo.numOfColumns, ctgTestColNum);
+  ASSERT_EQ(tableMeta->tableInfo.numOfTags, ctgTestTagNum);
+  ASSERT_EQ(tableMeta->tableInfo.precision, 1 + 1);
+  ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
+
+  tfree(tableMeta);
+  
+  catalogDestroy();
+}
+
+
+
 TEST(tableDistVgroup, normalTable) {
   struct SCatalog *pCtg = NULL;
   void            *mockPointer = (void *)0x1;
@@ -1109,7 +1297,7 @@ TEST(dbVgroup, getSetDbVgroupCase) {
   taosArrayDestroy(vgList);
 
   ctgTestBuildDBVgroup(&dbVgroup);
-  code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, dbVgroup);
+  code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, ctgTestDbId, dbVgroup);
   ASSERT_EQ(code, 0);
 
   code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
@@ -1128,7 +1316,7 @@ TEST(dbVgroup, getSetDbVgroupCase) {
   catalogDestroy();
 }
 
-TEST(multiThread, getSetDbVgroupCase) {
+TEST(multiThread, getSetRmSameDbVgroup) {
   struct SCatalog *pCtg = NULL;
   void            *mockPointer = (void *)0x1;
   SVgroupInfo      vgInfo = {0};
@@ -1159,10 +1347,10 @@ TEST(multiThread, getSetDbVgroupCase) {
   pthread_attr_init(&thattr);
 
   pthread_t thread1, thread2;
-  pthread_create(&(thread1), &thattr, ctgTestSetDbVgroupThread, pCtg);
+  pthread_create(&(thread1), &thattr, ctgTestSetSameDbVgroupThread, pCtg);
 
   sleep(1);
-  pthread_create(&(thread1), &thattr, ctgTestGetDbVgroupThread, pCtg);
+  pthread_create(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg);
 
   while (true) {
     if (ctgTestDeadLoop) {
@@ -1179,6 +1367,58 @@ TEST(multiThread, getSetDbVgroupCase) {
   catalogDestroy();
 }
 
+TEST(multiThread, getSetRmDiffDbVgroup) {
+  struct SCatalog *pCtg = NULL;
+  void            *mockPointer = (void *)0x1;
+  SVgroupInfo      vgInfo = {0};
+  SVgroupInfo     *pvgInfo = NULL;
+  SDBVgroupInfo    dbVgroup = {0};
+  SArray          *vgList = NULL;
+  ctgTestStop = false;
+
+  ctgTestInitLogFile();
+
+  ctgTestSetPrepareDbVgroups();
+
+  initQueryModuleMsgHandle();
+
+  // sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet);
+
+  int32_t code = catalogInit(NULL);
+  ASSERT_EQ(code, 0);
+
+  code = catalogGetHandle(ctgTestClusterId, &pCtg);
+  ASSERT_EQ(code, 0);
+
+  SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
+  strcpy(n.dbname, "db1");
+  strcpy(n.tname, ctgTestTablename);
+
+  pthread_attr_t thattr;
+  pthread_attr_init(&thattr);
+
+  pthread_t thread1, thread2;
+  pthread_create(&(thread1), &thattr, ctgTestSetDiffDbVgroupThread, pCtg);
+
+  sleep(1);
+  pthread_create(&(thread2), &thattr, ctgTestGetDbVgroupThread, pCtg);
+
+  while (true) {
+    if (ctgTestDeadLoop) {
+      sleep(1);
+    } else {
+      sleep(ctgTestMTRunSec);
+      break;
+    }
+  }
+
+  ctgTestStop = true;
+  sleep(1);
+
+  catalogDestroy();
+}
+
+
 
 TEST(multiThread, ctableMeta) {
   struct SCatalog *pCtg = NULL;

source/libs/function/src/functionMgt.c

@@ -78,3 +78,11 @@ FuncDef setExecFuncs(FuncDef def, FExecGetEnv getEnv, FExecInit init, FExecProce
 int32_t registerFunc(FuncDef func) {
 
 }
+
+int32_t fmGetFuncResultType(FuncMgtHandle handle, SFunctionNode* pFunc) {
+  return TSDB_CODE_SUCCESS;
+}
+
+bool fmIsAggFunc(int32_t funcId) {
+  return false;
+}

source/libs/parser/inc/astCreateFuncs.h

@@ -27,12 +27,10 @@ extern "C" {
 
 extern SToken nil_token;
 
-typedef struct STargetExprNode {
-  ENodeType nodeType;
-  char* p;
-  uint32_t n;
-  SNode* pNode;
-} STargetExprNode;
+SNode* createRawExprNode(SAstCreateContext* pCxt, const SToken* pToken, SNode* pNode);
+SNode* createRawExprNodeExt(SAstCreateContext* pCxt, const SToken* pStart, const SToken* pEnd, SNode* pNode);
+SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode);
+SToken getTokenFromRawExprNode(SAstCreateContext* pCxt, SNode* pNode);
 
 SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode);
 SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode);

source/libs/parser/inc/astToMsg.h

@@ -10,8 +10,8 @@ SCreateAcctReq* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, in
 SDropUserReq* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen);
 SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext* pParseCtx, SMsgBuf* pMsgBuf);
 SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseContext *pCtx, SMsgBuf* pMsgBuf);
-SMCreateStbReq*   buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf);
-SMDropStbReq* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf);
+char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf);
+char* buildDropStableReq(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf);
 SCreateDnodeReq *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf);
 SDropDnodeReq *buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf);
 

source/libs/parser/inc/new_sql.y

@@ -67,19 +67,19 @@ cmd ::= SHOW DATABASES.
 cmd ::= query_expression(A).                                                      { PARSER_TRACE; pCxt->pRootNode = A; }
 
 /************************************************ literal *************************************************************/
-literal(A) ::= NK_INTEGER(B).                                                     { PARSER_TRACE; A = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); }
-literal(A) ::= NK_FLOAT(B).                                                       { PARSER_TRACE; A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); }
-literal(A) ::= NK_STRING(B).                                                      { PARSER_TRACE; A = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); }
-literal(A) ::= NK_BOOL(B).                                                        { PARSER_TRACE; A = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B); }
-literal(A) ::= TIMESTAMP NK_STRING(B).                                            { PARSER_TRACE; A = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &B); }
+literal(A) ::= NK_INTEGER(B).                                                     { PARSER_TRACE; A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); }
+literal(A) ::= NK_FLOAT(B).                                                       { PARSER_TRACE; A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B)); }
+literal(A) ::= NK_STRING(B).                                                      { PARSER_TRACE; A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B)); }
+literal(A) ::= NK_BOOL(B).                                                        { PARSER_TRACE; A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B)); }
+literal(A) ::= TIMESTAMP(B) NK_STRING(C).                                         { PARSER_TRACE; A = createRawExprNodeExt(pCxt, &B, &C, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &C)); }
 literal(A) ::= duration_literal(B).                                               { PARSER_TRACE; A = B; }
 
-duration_literal(A) ::= NK_VARIABLE(B).                                           { PARSER_TRACE; A = createDurationValueNode(pCxt, &B); }
+duration_literal(A) ::= NK_VARIABLE(B).                                           { PARSER_TRACE; A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); }
 
 %type literal_list                                                                { SNodeList* }
 %destructor literal_list                                                          { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); }
-literal_list(A) ::= literal(B).                                                   { PARSER_TRACE; A = createNodeList(pCxt, B); }
-literal_list(A) ::= literal_list(B) NK_COMMA literal(C).                          { PARSER_TRACE; A = addNodeToList(pCxt, B, C); }
+literal_list(A) ::= literal(B).                                                   { PARSER_TRACE; A = createNodeList(pCxt, releaseRawExprNode(pCxt, B)); }
+literal_list(A) ::= literal_list(B) NK_COMMA literal(C).                          { PARSER_TRACE; A = addNodeToList(pCxt, B, releaseRawExprNode(pCxt, C)); }
 
 /************************************************ names and identifiers ***********************************************/
 %type db_name                                                                     { SToken }
@@ -111,37 +111,70 @@ expression(A) ::= literal(B).
 //expression(A) ::= NK_QUESTION(B).                                                 { PARSER_TRACE; A = B; }
 //expression(A) ::= pseudo_column(B).                                               { PARSER_TRACE; A = B; }
 expression(A) ::= column_reference(B).                                            { PARSER_TRACE; A = B; }
-expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP.                { PARSER_TRACE; A = createFunctionNode(pCxt, &B, C); }
+expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D).             { PARSER_TRACE; A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); }
 //expression(A) ::= cast_expression(B).                                             { PARSER_TRACE; A = B; }
 //expression(A) ::= case_expression(B).                                             { PARSER_TRACE; A = B; }
 expression(A) ::= subquery(B).                                                    { PARSER_TRACE; A = B; }
-expression(A) ::= NK_LP expression(B) NK_RP.                                      { PARSER_TRACE; A = B; }
-expression(A) ::= NK_PLUS expression(B).                                          { PARSER_TRACE; A = B; }
-expression(A) ::= NK_MINUS expression(B).                                         { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_SUB, B, NULL); }
-expression(A) ::= expression(B) NK_PLUS expression(C).                            { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_ADD, B, C); }
-expression(A) ::= expression(B) NK_MINUS expression(C).                           { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_SUB, B, C); }
-expression(A) ::= expression(B) NK_STAR expression(C).                            { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_MULTI, B, C); }
-expression(A) ::= expression(B) NK_SLASH expression(C).                           { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_DIV, B, C); }
-expression(A) ::= expression(B) NK_REM expression(C).                             { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_MOD, B, C); }
+expression(A) ::= NK_LP(B) expression(C) NK_RP(D).                                { PARSER_TRACE; A = createRawExprNodeExt(pCxt, &B, &D, releaseRawExprNode(pCxt, C)); }
+expression(A) ::= NK_PLUS(B) expression(C).                                       {
+                                                                                    PARSER_TRACE;
+                                                                                    SToken t = getTokenFromRawExprNode(pCxt, C);
+                                                                                    A = createRawExprNodeExt(pCxt, &B, &t, releaseRawExprNode(pCxt, C));
+                                                                                  }
+expression(A) ::= NK_MINUS(B) expression(C).                                      {
+                                                                                    PARSER_TRACE;
+                                                                                    SToken t = getTokenFromRawExprNode(pCxt, C);
+                                                                                    A = createRawExprNodeExt(pCxt, &B, &t, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, C), NULL));
+                                                                                  }
+expression(A) ::= expression(B) NK_PLUS expression(C).                            {
+                                                                                    PARSER_TRACE; 
+                                                                                    SToken s = getTokenFromRawExprNode(pCxt, B);
+                                                                                    SToken e = getTokenFromRawExprNode(pCxt, C);
+                                                                                    A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); 
+                                                                                  }
+expression(A) ::= expression(B) NK_MINUS expression(C).                           {
+                                                                                    PARSER_TRACE; 
+                                                                                    SToken s = getTokenFromRawExprNode(pCxt, B);
+                                                                                    SToken e = getTokenFromRawExprNode(pCxt, C);
+                                                                                    A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); 
+                                                                                  }
+expression(A) ::= expression(B) NK_STAR expression(C).                            {
+                                                                                    PARSER_TRACE; 
+                                                                                    SToken s = getTokenFromRawExprNode(pCxt, B);
+                                                                                    SToken e = getTokenFromRawExprNode(pCxt, C);
+                                                                                    A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); 
+                                                                                  }
+expression(A) ::= expression(B) NK_SLASH expression(C).                           {
+                                                                                    PARSER_TRACE; 
+                                                                                    SToken s = getTokenFromRawExprNode(pCxt, B);
+                                                                                    SToken e = getTokenFromRawExprNode(pCxt, C);
+                                                                                    A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); 
+                                                                                  }
+expression(A) ::= expression(B) NK_REM expression(C).                             {
+                                                                                    PARSER_TRACE; 
+                                                                                    SToken s = getTokenFromRawExprNode(pCxt, B);
+                                                                                    SToken e = getTokenFromRawExprNode(pCxt, C);
+                                                                                    A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); 
+                                                                                  }
 
 %type expression_list                                                             { SNodeList* }
 %destructor expression_list                                                       { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); }
-expression_list(A) ::= expression(B).                                             { PARSER_TRACE; A = createNodeList(pCxt, B); }
-expression_list(A) ::= expression_list(B) NK_COMMA expression(C).                 { PARSER_TRACE; A = addNodeToList(pCxt, B, C); }
+expression_list(A) ::= expression(B).                                             { PARSER_TRACE; A = createNodeList(pCxt, releaseRawExprNode(pCxt, B)); }
+expression_list(A) ::= expression_list(B) NK_COMMA expression(C).                 { PARSER_TRACE; A = addNodeToList(pCxt, B, releaseRawExprNode(pCxt, C)); }
 
-column_reference(A) ::= column_name(B).                                           { PARSER_TRACE; A = createColumnNode(pCxt, NULL, &B); }
-column_reference(A) ::= table_name(B) NK_DOT column_name(C).                      { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); }
+column_reference(A) ::= column_name(B).                                           { PARSER_TRACE; A = createRawExprNode(pCxt, &B, createColumnNode(pCxt, NULL, &B)); }
+column_reference(A) ::= table_name(B) NK_DOT column_name(C).                      { PARSER_TRACE; A = createRawExprNodeExt(pCxt, &B, &C, createColumnNode(pCxt, &B, &C)); }
 
 //pseudo_column(A) ::= NK_NOW.                                                      { PARSER_TRACE; A = createFunctionNode(pCxt, NULL, NULL); }
 
 /************************************************ predicate ***********************************************************/
-predicate(A) ::= expression(B) compare_op(C) expression(D).                       { PARSER_TRACE; A = createOperatorNode(pCxt, C, B, D); }
+predicate(A) ::= expression(B) compare_op(C) expression(D).                       { PARSER_TRACE; A = createOperatorNode(pCxt, C, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D)); }
 //predicate(A) ::= expression(B) compare_op sub_type expression(B).
-predicate(A) ::= expression(B) BETWEEN expression(C) AND expression(D).           { PARSER_TRACE; A = createBetweenAnd(pCxt, B, C, D); }
-predicate(A) ::= expression(B) NOT BETWEEN expression(C) AND expression(D).       { PARSER_TRACE; A = createNotBetweenAnd(pCxt, C, B, D); }
-predicate(A) ::= expression(B) IS NULL.                                           { PARSER_TRACE; A = createIsNullCondNode(pCxt, B, true); }
-predicate(A) ::= expression(B) IS NOT NULL.                                       { PARSER_TRACE; A = createIsNullCondNode(pCxt, B, false); }
-predicate(A) ::= expression(B) in_op(C) in_predicate_value(D).                    { PARSER_TRACE; A = createOperatorNode(pCxt, C, B, D); }
+predicate(A) ::= expression(B) BETWEEN expression(C) AND expression(D).           { PARSER_TRACE; A = createBetweenAnd(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, D)); }
+predicate(A) ::= expression(B) NOT BETWEEN expression(C) AND expression(D).       { PARSER_TRACE; A = createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D)); }
+predicate(A) ::= expression(B) IS NULL.                                           { PARSER_TRACE; A = createIsNullCondNode(pCxt, releaseRawExprNode(pCxt, B), true); }
+predicate(A) ::= expression(B) IS NOT NULL.                                       { PARSER_TRACE; A = createIsNullCondNode(pCxt, releaseRawExprNode(pCxt, B), false); }
+predicate(A) ::= expression(B) in_op(C) in_predicate_value(D).                    { PARSER_TRACE; A = createOperatorNode(pCxt, C, releaseRawExprNode(pCxt, B), D); }
 
 %type compare_op                                                                  { EOperatorType }
 %destructor compare_op                                                            { PARSER_DESTRUCTOR_TRACE; }
@@ -186,7 +219,7 @@ table_reference(A) ::= joined_table(B).
 
 table_primary(A) ::= table_name(B) alias_opt(C).                                  { PARSER_TRACE; A = createRealTableNode(pCxt, NULL, &B, &C); }
 table_primary(A) ::= db_name(B) NK_DOT table_name(C) alias_opt(D).                { PARSER_TRACE; A = createRealTableNode(pCxt, &B, &C, &D); }
-table_primary(A) ::= subquery(B) alias_opt(C).                                    { PARSER_TRACE; A = createTempTableNode(pCxt, B, &C); }
+table_primary(A) ::= subquery(B) alias_opt(C).                                    { PARSER_TRACE; A = createTempTableNode(pCxt, releaseRawExprNode(pCxt, B), &C); }
 table_primary(A) ::= parenthesized_joined_table(B).                               { PARSER_TRACE; A = B; }
 
 %type alias_opt                                                                   { SToken }
@@ -236,9 +269,13 @@ select_list(A) ::= select_sublist(B).
 select_sublist(A) ::= select_item(B).                                             { PARSER_TRACE; A = createNodeList(pCxt, B); }
 select_sublist(A) ::= select_sublist(B) NK_COMMA select_item(C).                  { PARSER_TRACE; A = addNodeToList(pCxt, B, C); }
 
-select_item(A) ::= expression(B).                                                 { PARSER_TRACE; A = B; }
-select_item(A) ::= expression(B) column_alias(C).                                 { PARSER_TRACE; A = setProjectionAlias(pCxt, B, &C); }
-select_item(A) ::= expression(B) AS column_alias(C).                              { PARSER_TRACE; A = setProjectionAlias(pCxt, B, &C); }
+select_item(A) ::= expression(B).                                                 {
+                                                                                    PARSER_TRACE;
+                                                                                    SToken t = getTokenFromRawExprNode(pCxt, B);
+                                                                                    A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &t);
+                                                                                  }
+select_item(A) ::= expression(B) column_alias(C).                                 { PARSER_TRACE; A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); }
+select_item(A) ::= expression(B) AS column_alias(C).                              { PARSER_TRACE; A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); }
 select_item(A) ::= table_name(B) NK_DOT NK_STAR(C).                               { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); }
 
 where_clause_opt(A) ::= .                                                         { PARSER_TRACE; A = NULL; }
@@ -251,8 +288,8 @@ partition_by_clause_opt(A) ::= PARTITION BY expression_list(B).
 
 twindow_clause_opt(A) ::= .                                                       { PARSER_TRACE; A = NULL; }
 twindow_clause_opt(A) ::=
-  SESSION NK_LP column_reference(B) NK_COMMA NK_INTEGER(C) NK_RP.                 { PARSER_TRACE; A = createSessionWindowNode(pCxt, B, &C); }
-twindow_clause_opt(A) ::= STATE_WINDOW NK_LP column_reference(B) NK_RP.           { PARSER_TRACE; A = createStateWindowNode(pCxt, B); }
+  SESSION NK_LP column_reference(B) NK_COMMA NK_INTEGER(C) NK_RP.                 { PARSER_TRACE; A = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, B), &C); }
+twindow_clause_opt(A) ::= STATE_WINDOW NK_LP column_reference(B) NK_RP.           { PARSER_TRACE; A = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, B)); }
 twindow_clause_opt(A) ::=
   INTERVAL NK_LP duration_literal(B) NK_RP sliding_opt(C) fill_opt(D).            { PARSER_TRACE; A = createIntervalWindowNode(pCxt, B, NULL, C, D); }
 twindow_clause_opt(A) ::=
@@ -317,7 +354,7 @@ limit_clause_opt(A) ::= LIMIT NK_INTEGER(B) OFFSET NK_INTEGER(C).
 limit_clause_opt(A) ::= LIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B).               { PARSER_TRACE; A = createLimitNode(pCxt, &B, &C); }
 
 /************************************************ subquery ************************************************************/
-subquery(A) ::= NK_LP query_expression(B) NK_RP.                                  { PARSER_TRACE; A = B; }
+subquery(A) ::= NK_LP(B) query_expression(C) NK_RP(D).                                  { PARSER_TRACE; A = createRawExprNodeExt(pCxt, &B, &D, C); }
 
 /************************************************ search_condition ****************************************************/
 search_condition(A) ::= boolean_value_expression(B).                              { PARSER_TRACE; A = B; }
@@ -330,7 +367,7 @@ sort_specification_list(A) ::=
   sort_specification_list(B) NK_COMMA sort_specification(C).                      { PARSER_TRACE; A = addNodeToList(pCxt, B, C); }
 
 sort_specification(A) ::= 
-  expression(B) ordering_specification_opt(C) null_ordering_opt(D).               { PARSER_TRACE; A = createOrderByExprNode(pCxt, B, C, D); }
+  expression(B) ordering_specification_opt(C) null_ordering_opt(D).               { PARSER_TRACE; A = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, B), C, D); }
 
 %type ordering_specification_opt EOrder
 %destructor ordering_specification_opt                                            { PARSER_DESTRUCTOR_TRACE; }

source/libs/parser/inc/sql.y

@@ -820,7 +820,7 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) MODIFY COLUMN columnlist(A).     {
 //////////////////////////////////ALTER TAGS statement/////////////////////////////////////
 cmd ::= ALTER TABLE ids(X) cpxName(Y) ADD TAG columnlist(A).        {
     X.n += Y.n;
-    SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, -1);
+    SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG, -1);
     setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
 cmd ::= ALTER TABLE ids(X) cpxName(Z) DROP TAG ids(Y).          {
@@ -829,7 +829,7 @@ cmd ::= ALTER TABLE ids(X) cpxName(Z) DROP TAG ids(Y).          {
     toTSDBType(Y.type);
     SArray* A = tListItemAppendToken(NULL, &Y, -1);
 
-    SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, -1);
+    SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG, -1);
     setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
 
@@ -889,7 +889,7 @@ cmd ::= ALTER STABLE ids(X) cpxName(F) MODIFY COLUMN columnlist(A).     {
 //////////////////////////////////ALTER TAGS statement/////////////////////////////////////
 cmd ::= ALTER STABLE ids(X) cpxName(Y) ADD TAG columnlist(A).        {
     X.n += Y.n;
-    SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, TSDB_SUPER_TABLE);
+    SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG, TSDB_SUPER_TABLE);
     setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
 cmd ::= ALTER STABLE ids(X) cpxName(Z) DROP TAG ids(Y).          {
@@ -898,7 +898,7 @@ cmd ::= ALTER STABLE ids(X) cpxName(Z) DROP TAG ids(Y).          {
     toTSDBType(Y.type);
     SArray* A = tListItemAppendToken(NULL, &Y, -1);
 
-    SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, TSDB_SUPER_TABLE);
+    SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG, TSDB_SUPER_TABLE);
     setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
 

source/libs/parser/src/astCreateFuncs.c

@@ -24,6 +24,14 @@
     } \
   } while (0)
 
+#define CHECK_RAW_EXPR_NODE(node) \
+  do { \
+    if (NULL == (node) || QUERY_NODE_RAW_EXPR != nodeType(node)) { \
+      pCxt->valid = false; \
+      return NULL; \
+    } \
+  } while (0)
+
 SToken nil_token = { .type = TK_NIL, .n = 0, .z = NULL };
 
 static bool checkDbName(SAstCreateContext* pCxt, const SToken* pDbName) {
@@ -50,6 +58,37 @@ static bool checkColumnName(SAstCreateContext* pCxt, const SToken* pColumnName)
   return pCxt->valid;
 }
 
+SNode* createRawExprNode(SAstCreateContext* pCxt, const SToken* pToken, SNode* pNode) {
+  SRawExprNode* target = (SRawExprNode*)nodesMakeNode(QUERY_NODE_RAW_EXPR);
+  CHECK_OUT_OF_MEM(target);
+  target->p = pToken->z;
+  target->n = pToken->n;
+  target->pNode = pNode;
+  return (SNode*)target;
+}
+
+SNode* createRawExprNodeExt(SAstCreateContext* pCxt, const SToken* pStart, const SToken* pEnd, SNode* pNode) {
+  SRawExprNode* target = (SRawExprNode*)nodesMakeNode(QUERY_NODE_RAW_EXPR);
+  CHECK_OUT_OF_MEM(target);
+  target->p = pStart->z;
+  target->n = (pEnd->z + pEnd->n) - pStart->z;
+  target->pNode = pNode;
+  return (SNode*)target;
+}
+
+SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) {
+  CHECK_RAW_EXPR_NODE(pNode);
+  SNode* tmp = ((SRawExprNode*)pNode)->pNode;
+  tfree(pNode);
+  return tmp;
+}
+
+SToken getTokenFromRawExprNode(SAstCreateContext* pCxt, SNode* pNode) {
+  SRawExprNode* target = (SRawExprNode*)pNode;
+  SToken t = { .type = 0, .z = target->p, .n = target->n};
+  return t;
+}
+
 SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode) {
   SNodeList* list = nodesMakeList();
   CHECK_OUT_OF_MEM(list);
@@ -79,14 +118,22 @@ SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken*
   val->literal = strndup(pLiteral->z, pLiteral->n);
   CHECK_OUT_OF_MEM(val->literal);
   val->node.resType.type = dataType;
-  val->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
+  val->node.resType.bytes = tDataTypes[dataType].bytes;
+  if (TSDB_DATA_TYPE_TIMESTAMP == dataType) {
+    val->node.resType.precision = TSDB_TIME_PRECISION_MILLI;
+  }
   return (SNode*)val;
 }
 
 SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral) {
   SValueNode* val = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
   CHECK_OUT_OF_MEM(val);
-  // todo
+  val->literal = strndup(pLiteral->z, pLiteral->n);
+  CHECK_OUT_OF_MEM(val->literal);
+  val->isDuration = true;
+  val->node.resType.type = TSDB_DATA_TYPE_BIGINT;
+  val->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;
+  val->node.resType.precision = TSDB_TIME_PRECISION_MILLI;
   return (SNode*)val;
 }
 

source/libs/parser/src/astGenerator.c

@@ -610,7 +610,7 @@ SAlterTableInfo *tSetAlterTableInfo(SToken *pTableName, SArray *pCols, SArray *p
   pAlterTable->type = type;
   pAlterTable->tableType = tableType;
 
-  if (type == TSDB_ALTER_TABLE_ADD_COLUMN || type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN || type == TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES || type == TSDB_ALTER_TABLE_UPDATE_TAG_BYTES) {
+  if (type == TSDB_ALTER_TABLE_ADD_COLUMN || type == TSDB_ALTER_TABLE_ADD_TAG || type == TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES || type == TSDB_ALTER_TABLE_UPDATE_TAG_BYTES) {
     pAlterTable->pAddColumns = pCols;
     assert(pVals == NULL);
   } else {

source/libs/parser/src/dCDAstProcess.c

@@ -924,13 +924,13 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch
         goto _error;
       }
 
-      pDcl->pMsg = (char*)buildCreateStbMsg(pCreateTable, &pDcl->msgLen, pCtx, pMsgBuf);
+      pDcl->pMsg = buildCreateStbReq(pCreateTable, &pDcl->msgLen, pCtx, pMsgBuf);
       pDcl->msgType = TDMT_MND_CREATE_STB;
       break;
     }
 
     case TSDB_SQL_DROP_TABLE: {
-      pDcl->pMsg = (char*)buildDropStableMsg(pInfo, &pDcl->msgLen, pCtx, pMsgBuf);
+      pDcl->pMsg = buildDropStableReq(pInfo, &pDcl->msgLen, pCtx, pMsgBuf);
       if (pDcl->pMsg == NULL) {
         goto _error;
       }

source/libs/sync/CMakeLists.txt

+
 aux_source_directory(src SYNC_SRC)
 add_library(sync ${SYNC_SRC})