Merge remote-tracking branch 'origin/3.0' into feature/tfs

.devcontainer/devcontainer.json

@@ -26,7 +26,8 @@
 		"eamodio.gitlens",
 		"matepek.vscode-catch2-test-adapter",
 		"spmeesseman.vscode-taskexplorer",
-		"cschlosser.doxdocgen"
+		"cschlosser.doxdocgen",
+		"urosvujosevic.explorer-manager"
 	],
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	// "forwardPorts": [],

cmake/bdb_CMakeLists.txt.in

@@ -6,7 +6,7 @@ ExternalProject_Add(bdb
         SOURCE_DIR "${CMAKE_CONTRIB_DIR}/bdb"
         BINARY_DIR "${CMAKE_CONTRIB_DIR}/bdb"
         #BUILD_IN_SOURCE TRUE
-        CONFIGURE_COMMAND "./dist/configure"
+        CONFIGURE_COMMAND COMMAND ./dist/configure --enable-debug
         BUILD_COMMAND "$(MAKE)"
         INSTALL_COMMAND ""
         TEST_COMMAND ""

include/client/taos.h

@@ -193,8 +193,7 @@ DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr);
 DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList);
 DLL_EXPORT TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision);
 
-
-DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen);
+DLL_EXPORT TAOS_RES *taos_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen);
 
 #ifdef __cplusplus
 }

include/common/tmsg.h

@@ -68,6 +68,14 @@ typedef uint16_t tmsg_t;
 #define TSDB_IE_TYPE_DNODE_EXT 6
 #define TSDB_IE_TYPE_DNODE_STATE 7
 
+typedef enum {
+  HEARTBEAT_TYPE_MQ    = 0,
+  HEARTBEAT_TYPE_QUERY = 1,
+  // types can be added here
+  //
+  HEARTBEAT_TYPE_MAX
+} EHbType;
+
 typedef enum _mgmt_table {
   TSDB_MGMT_TABLE_START,
   TSDB_MGMT_TABLE_ACCT,
@@ -147,7 +155,7 @@ typedef struct {
 
 typedef struct {
   SClientHbKey connKey;
-  SHashObj*    info;  // hash<Slv.key, Sklv>
+  SHashObj*    info;  // hash<Skv.key, Skv>
 } SClientHbReq;
 
 typedef struct {
@@ -173,7 +181,10 @@ static FORCE_INLINE uint32_t hbKeyHashFunc(const char* key, uint32_t keyLen) {
 }
 
 int   tSerializeSClientHbReq(void** buf, const SClientHbReq* pReq);
-void* tDeserializeClientHbReq(void* buf, SClientHbReq* pReq);
+void* tDeserializeSClientHbReq(void* buf, SClientHbReq* pReq);
+
+int   tSerializeSClientHbRsp(void** buf, const SClientHbRsp* pRsp);
+void* tDeserializeSClientHbRsp(void* buf, SClientHbRsp* pRsp);
 
 static FORCE_INLINE void  tFreeClientHbReq(void *pReq) {
   SClientHbReq* req = (SClientHbReq*)pReq;
@@ -182,14 +193,17 @@ static FORCE_INLINE void  tFreeClientHbReq(void *pReq) {
 }
 
 int   tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pReq);
-void* tDeserializeClientHbBatchReq(void* buf, SClientHbBatchReq* pReq);
+void* tDeserializeSClientHbBatchReq(void* buf, SClientHbBatchReq* pReq);
 
 static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq) {
   SClientHbBatchReq *req = (SClientHbBatchReq*)pReq;
-  taosArrayDestroyEx(req->reqs, tFreeClientHbReq);
+  //taosArrayDestroyEx(req->reqs, tFreeClientHbReq);
   free(pReq);
 }
 
+int   tSerializeSClientHbBatchRsp(void** buf, const SClientHbBatchRsp* pBatchRsp);
+void* tDeserializeSClientHbBatchRsp(void* buf, SClientHbBatchRsp* pBatchRsp);
+
 static FORCE_INLINE int taosEncodeSKv(void** buf, const SKv* pKv) {
   int tlen = 0;
   tlen += taosEncodeFixedI32(buf, pKv->keyLen);
@@ -220,6 +234,93 @@ static FORCE_INLINE void* taosDecodeSClientHbKey(void* buf, SClientHbKey* pKey)
   return buf;
 }
 
+typedef struct SMqHbVgInfo {
+  int32_t vgId;
+} SMqHbVgInfo;
+
+static FORCE_INLINE int taosEncodeSMqVgInfo(void** buf, const SMqHbVgInfo* pVgInfo) {
+  int tlen = 0;
+  tlen += taosEncodeFixedI32(buf, pVgInfo->vgId);
+  return tlen;
+}
+
+static FORCE_INLINE void* taosDecodeSMqVgInfo(void* buf, SMqHbVgInfo* pVgInfo) {
+  buf = taosDecodeFixedI32(buf, &pVgInfo->vgId);
+  return buf;
+}
+
+typedef struct SMqHbTopicInfo {
+  int32_t epoch;
+  int64_t topicUid;
+  char    name[TSDB_TOPIC_FNAME_LEN];
+  SArray* pVgInfo;
+} SMqHbTopicInfo;
+
+static FORCE_INLINE int taosEncodeSMqHbTopicInfoMsg(void** buf, const SMqHbTopicInfo* pTopicInfo) {
+  int tlen = 0;
+  tlen += taosEncodeFixedI32(buf, pTopicInfo->epoch);
+  tlen += taosEncodeFixedI64(buf, pTopicInfo->topicUid);
+  tlen += taosEncodeString(buf, pTopicInfo->name);
+  int32_t sz = taosArrayGetSize(pTopicInfo->pVgInfo);
+  tlen += taosEncodeFixedI32(buf, sz);
+  for (int32_t i = 0; i < sz; i++) {
+    SMqHbVgInfo* pVgInfo = (SMqHbVgInfo*)taosArrayGet(pTopicInfo->pVgInfo, i);
+    tlen += taosEncodeSMqVgInfo(buf, pVgInfo);
+  }
+  return tlen;
+}
+
+static FORCE_INLINE void* taosDecodeSMqHbTopicInfoMsg(void* buf, SMqHbTopicInfo* pTopicInfo) {
+  buf = taosDecodeFixedI32(buf, &pTopicInfo->epoch);
+  buf = taosDecodeFixedI64(buf, &pTopicInfo->topicUid);
+  buf = taosDecodeStringTo(buf, pTopicInfo->name);
+  int32_t sz;
+  buf = taosDecodeFixedI32(buf, &sz);
+  pTopicInfo->pVgInfo = taosArrayInit(sz, sizeof(SMqHbVgInfo));
+  for (int32_t i = 0; i < sz; i++) {
+    SMqHbVgInfo vgInfo;
+    buf = taosDecodeSMqVgInfo(buf, &vgInfo);
+    taosArrayPush(pTopicInfo->pVgInfo, &vgInfo);
+  }
+  return buf;
+}
+
+typedef struct SMqHbMsg {
+  int32_t  status;   // ask hb endpoint
+  int32_t  epoch;
+  int64_t  consumerId;
+  SArray*  pTopics;  // SArray<SMqHbTopicInfo>
+} SMqHbMsg;
+
+static FORCE_INLINE int taosEncodeSMqMsg(void** buf, const SMqHbMsg* pMsg) {
+  int tlen = 0;
+  tlen += taosEncodeFixedI32(buf, pMsg->status);
+  tlen += taosEncodeFixedI32(buf, pMsg->epoch);
+  tlen += taosEncodeFixedI64(buf, pMsg->consumerId);
+  int32_t sz = taosArrayGetSize(pMsg->pTopics);
+  tlen += taosEncodeFixedI32(buf, sz);
+  for (int i = 0; i < sz; i++) {
+    SMqHbTopicInfo* topicInfo = (SMqHbTopicInfo*)taosArrayGet(pMsg->pTopics, i);
+    tlen += taosEncodeSMqHbTopicInfoMsg(buf, topicInfo);
+  }
+  return tlen;
+}
+
+static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) {
+  buf = taosDecodeFixedI32(buf, &pMsg->status);
+  buf = taosDecodeFixedI32(buf, &pMsg->epoch);
+  buf = taosDecodeFixedI64(buf, &pMsg->consumerId);
+  int32_t sz;
+  buf = taosDecodeFixedI32(buf, &sz);
+  pMsg->pTopics = taosArrayInit(sz, sizeof(SMqHbTopicInfo));
+  for (int i = 0; i < sz; i++) {
+    SMqHbTopicInfo topicInfo;
+    buf = taosDecodeSMqHbTopicInfoMsg(buf, &topicInfo);
+    taosArrayPush(pMsg->pTopics, &topicInfo);
+  }
+  return buf;
+}
+
 typedef struct {
   int32_t vgId;
   char*   dbName;
@@ -359,6 +460,91 @@ static FORCE_INLINE void* taosDecodeSEpSet(void* buf, SEpSet* pEp) {
   return buf;
 }
 
+typedef struct SMqHbRsp {
+  int8_t status;    //idle or not
+  int8_t vnodeChanged;
+  int8_t epChanged; // should use new epset
+  int8_t reserved;
+  SEpSet epSet;
+} SMqHbRsp;
+
+static FORCE_INLINE int taosEncodeSMqHbRsp(void** buf, const SMqHbRsp* pRsp) {
+  int tlen = 0;
+  tlen += taosEncodeFixedI8(buf, pRsp->status);
+  tlen += taosEncodeFixedI8(buf, pRsp->vnodeChanged);
+  tlen += taosEncodeFixedI8(buf, pRsp->epChanged);
+  tlen += taosEncodeSEpSet(buf, &pRsp->epSet);
+  return tlen;
+}
+
+static FORCE_INLINE void* taosDecodeSMqHbRsp(void* buf, SMqHbRsp* pRsp) {
+  buf = taosDecodeFixedI8(buf, &pRsp->status);
+  buf = taosDecodeFixedI8(buf, &pRsp->vnodeChanged);
+  buf = taosDecodeFixedI8(buf, &pRsp->epChanged);
+  buf = taosDecodeSEpSet(buf, &pRsp->epSet);
+  return buf;
+}
+
+typedef struct SMqHbOneTopicBatchRsp {
+  char topicName[TSDB_TOPIC_FNAME_LEN];
+  SArray* rsps;  // SArray<SMqHbRsp>
+} SMqHbOneTopicBatchRsp;
+
+static FORCE_INLINE int taosEncodeSMqHbOneTopicBatchRsp(void** buf, const SMqHbOneTopicBatchRsp* pBatchRsp) {
+  int tlen = 0;
+  tlen += taosEncodeString(buf, pBatchRsp->topicName);
+  int32_t sz = taosArrayGetSize(pBatchRsp->rsps);
+  tlen += taosEncodeFixedI32(buf, sz);
+  for (int32_t i = 0; i < sz; i++) {
+    SMqHbRsp* pRsp = (SMqHbRsp*)taosArrayGet(pBatchRsp->rsps, i);
+    tlen += taosEncodeSMqHbRsp(buf, pRsp);
+  }
+  return tlen;
+}
+
+static FORCE_INLINE void* taosDecodeSMqHbOneTopicBatchRsp(void* buf, SMqHbOneTopicBatchRsp* pBatchRsp) {
+  int32_t sz;
+  buf = taosDecodeStringTo(buf, pBatchRsp->topicName);
+  buf = taosDecodeFixedI32(buf, &sz);
+  pBatchRsp->rsps = taosArrayInit(sz, sizeof(SMqHbRsp));
+  for (int32_t i = 0; i < sz; i++) {
+    SMqHbRsp rsp;
+    buf = taosDecodeSMqHbRsp(buf, &rsp);
+    buf = taosArrayPush(pBatchRsp->rsps, &rsp);
+  }
+  return buf;
+}
+
+typedef struct SMqHbBatchRsp {
+  int64_t consumerId; 
+  SArray* batchRsps; // SArray<SMqHbOneTopicBatchRsp>
+} SMqHbBatchRsp;
+
+static FORCE_INLINE int taosEncodeSMqHbBatchRsp(void** buf, const SMqHbBatchRsp* pBatchRsp) {
+  int tlen = 0;
+  tlen += taosEncodeFixedI64(buf, pBatchRsp->consumerId);
+  int32_t sz;
+  tlen += taosEncodeFixedI32(buf, sz);
+  for (int32_t i = 0; i < sz; i++) {
+    SMqHbOneTopicBatchRsp* pRsp = (SMqHbOneTopicBatchRsp*) taosArrayGet(pBatchRsp->batchRsps, i);
+    tlen += taosEncodeSMqHbOneTopicBatchRsp(buf, pRsp);
+  }
+  return tlen;
+}
+
+static FORCE_INLINE void* taosDecodeSMqHbBatchRsp(void* buf, SMqHbBatchRsp* pBatchRsp) {
+  buf = taosDecodeFixedI64(buf, &pBatchRsp->consumerId);
+  int32_t sz;
+  buf = taosDecodeFixedI32(buf, &sz);
+  pBatchRsp->batchRsps = taosArrayInit(sz, sizeof(SMqHbOneTopicBatchRsp));
+  for (int32_t i = 0; i < sz; i++) {
+    SMqHbOneTopicBatchRsp  rsp;
+    buf = taosDecodeSMqHbOneTopicBatchRsp(buf, &rsp);
+    buf = taosArrayPush(pBatchRsp->batchRsps, &rsp);
+  }
+  return buf;
+}
+
 typedef struct {
   int32_t acctId;
   int64_t clusterId;
@@ -993,6 +1179,13 @@ typedef struct {
   uint64_t taskId;
 } SSinkDataReq;
 
+typedef struct {
+  SMsgHead header;
+  uint64_t sId;
+  uint64_t queryId;
+  uint64_t taskId;
+} SQueryContinueReq;
+
 
 typedef struct {
   SMsgHead header;
@@ -1053,6 +1246,7 @@ typedef struct {
 typedef struct {
   int8_t igExists;
   char*  name;
+  char*  sql;
   char*  physicalPlan;
   char*  logicalPlan;
 } SCMCreateTopicReq;
@@ -1061,6 +1255,7 @@ static FORCE_INLINE int tSerializeSCMCreateTopicReq(void** buf, const SCMCreateT
   int tlen = 0;
   tlen += taosEncodeFixedI8(buf, pReq->igExists);
   tlen += taosEncodeString(buf, pReq->name);
+  tlen += taosEncodeString(buf, pReq->sql);
   tlen += taosEncodeString(buf, pReq->physicalPlan);
   tlen += taosEncodeString(buf, pReq->logicalPlan);
   return tlen;
@@ -1069,6 +1264,7 @@ static FORCE_INLINE int tSerializeSCMCreateTopicReq(void** buf, const SCMCreateT
 static FORCE_INLINE void* tDeserializeSCMCreateTopicReq(void* buf, SCMCreateTopicReq* pReq) {
   buf = taosDecodeFixedI8(buf, &(pReq->igExists));
   buf = taosDecodeString(buf, &(pReq->name));
+  buf = taosDecodeString(buf, &(pReq->sql));
   buf = taosDecodeString(buf, &(pReq->physicalPlan));
   buf = taosDecodeString(buf, &(pReq->logicalPlan));
   return buf;
@@ -1191,7 +1387,7 @@ typedef struct {
 } SMVSubscribeRsp;
 
 typedef struct {
-  char    name[TSDB_TOPIC_FNAME_LEN];
+  char    name[TSDB_TOPIC_NAME_LEN];
   int8_t  igExists;
   int32_t execLen;
   void*   executor;
@@ -1282,7 +1478,7 @@ typedef struct {
 typedef struct {
   SMsgHead head;
   char     name[TSDB_TABLE_FNAME_LEN];
-  int8_t   ignoreNotExists;
+  int64_t  suid;
 } SVDropTbReq;
 
 typedef struct {

include/common/tmsgdef.h

@@ -129,7 +129,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_VGROUP_LIST, "mnode-vgroup-list", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_KILL_QUERY, "mnode-kill-query", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_KILL_CONN, "mnode-kill-conn", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_MND_HEARTBEAT, "mnode-heartbeat", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_HEARTBEAT, "mnode-heartbeat", SClientHbBatchReq, SClientHbBatchRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_SHOW, "mnode-show", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_SHOW_RETRIEVE, "mnode-retrieve", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_STATUS, "mnode-status", NULL, NULL)
@@ -170,6 +170,8 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_DROP_TOPIC, "vnode-drop-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES, "vnode-show-tables", SVShowTablesReq, SVShowTablesRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES_FETCH, "vnode-show-tables-fetch", SVShowTablesFetchReq, SVShowTablesFetchRsp)
+  TD_DEF_MSG_TYPE(TDMT_VND_QUERY_CONTINUE, "vnode-query-continue", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SCHEDULE_DATA_SINK, "vnode-schedule-data-sink", NULL, NULL)
 
   TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp)
 

include/common/tname.h

@@ -25,14 +25,12 @@
 #define T_NAME_ACCT        0x1u
 #define T_NAME_DB          0x2u
 #define T_NAME_TABLE       0x4u
-#define T_NAME_TOPIC       0x8u
 
 typedef struct SName {
   uint8_t type;  //db_name_t, table_name_t
   int32_t acctId;
   char    dbname[TSDB_DB_NAME_LEN];
   char    tname[TSDB_TABLE_NAME_LEN];
-  char    topicName[TSDB_TOPIC_NAME_LEN];
 } SName;
 
 int32_t tNameExtractFullName(const SName* name, char* dst);

include/libs/executor/dataSinkMgt.h

@@ -48,7 +48,7 @@ typedef struct SOutputData {
   int8_t  compressed;
   char*   pData;
   bool    queryEnd;
-  bool    needSchedule;
+  int32_t scheduleJobNo;
   int32_t bufStatus;
   int64_t useconds;
   int8_t  precision;

include/libs/parser/parser.h

@@ -44,7 +44,7 @@ typedef struct SParseContext {
  * @param msg      extended error message if exists.
  * @return         error code
  */
-int32_t qParseQuerySql(SParseContext* pContext, SQueryNode** pQuery);
+int32_t qParseQuerySql(SParseContext* pContext, SQueryNode** pQueryNode);
 
 /**
  * Return true if it is a ddl/dcl sql statement

include/libs/planner/planner.h

@@ -95,6 +95,7 @@ typedef struct SScanPhyNode {
   int8_t      tableType;
   int32_t     order;         // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC
   int32_t     count;         // repeat count
+  int32_t     reverse;       // reverse scan count
 } SScanPhyNode;
 
 typedef SScanPhyNode SSystemTableScanPhyNode;
@@ -119,6 +120,25 @@ typedef struct SExchangePhyNode {
   SArray     *pSrcEndPoints;  // SEpAddr, scheduler fill by calling qSetSuplanExecutionNode
 } SExchangePhyNode;
 
+typedef enum EAggAlgo {
+	AGG_ALGO_PLAIN = 1, // simple agg across all input rows
+	AGG_ALGO_SORTED,    // grouped agg, input must be sorted
+	AGG_ALGO_HASHED     // grouped agg, use internal hashtable
+} EAggAlgo;
+
+typedef enum EAggSplit {
+	AGG_SPLIT_PRE = 1, // first level agg, maybe don't need calculate the final result
+	AGG_SPLIT_FINAL    // second level agg, must calculate the final result
+} EAggSplit;
+
+typedef struct SAggPhyNode {
+  SPhyNode    node;
+  EAggAlgo    aggAlgo;  // algorithm used by agg operator 
+  EAggSplit   aggSplit; // distributed splitting mode
+  SArray     *pExprs;   // SExprInfo list, these are expression list of group_by_clause and parameter expression of aggregate function
+  SArray     *pGroupByList; // SColIndex list, but these must be column node
+} SAggPhyNode;
+
 typedef struct SSubplanId {
   uint64_t queryId;
   uint64_t templateId;

include/libs/planner/plannerOp.h

@@ -30,7 +30,7 @@ OP_ENUM_MACRO(TagScan)
 OP_ENUM_MACRO(SystemTableScan)
 OP_ENUM_MACRO(Aggregate)
 OP_ENUM_MACRO(Project)
-OP_ENUM_MACRO(Groupby)
+// OP_ENUM_MACRO(Groupby)
 OP_ENUM_MACRO(Limit)
 OP_ENUM_MACRO(SLimit)
 OP_ENUM_MACRO(TimeWindow)

include/libs/qworker/qworker.h

@@ -22,9 +22,18 @@ extern "C" {
 
 #include "trpc.h"
 
+
+enum {
+  NODE_TYPE_VNODE = 1,
+  NODE_TYPE_QNODE,
+  NODE_TYPE_SNODE,
+};
+
+
+
 typedef struct SQWorkerCfg {
   uint32_t maxSchedulerNum;
-  uint32_t maxResCacheNum;
+  uint32_t maxTaskNum;
   uint32_t maxSchTaskNum;
 } SQWorkerCfg;
 
@@ -39,11 +48,17 @@ typedef struct {
   uint64_t numOfErrors;
 } SQWorkerStat;
 
+typedef int32_t (*putReqToQueryQFp)(void *, struct SRpcMsg *);
 
-int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt);
+
+int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, putReqToQueryQFp fp);
 
 int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
 
+int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
+
+int32_t qWorkerProcessDataSinkMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
+
 int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);
 
 int32_t qWorkerProcessStatusMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg);

include/libs/scheduler/scheduler.h

@@ -59,6 +59,11 @@ typedef struct SQueryResult {
   char    *msg;
 } SQueryResult;
 
+typedef struct STaskInfo {
+  SQueryNodeAddr addr;
+  SSubQueryMsg  *msg;
+} STaskInfo;
+
 int32_t schedulerInit(SSchedulerCfg *cfg);
 
 /**
@@ -101,6 +106,17 @@ void scheduleFreeJob(void *pJob);
 
 void schedulerDestroy(void);
 
+/**
+ * convert dag to task list
+ * @param pDag
+ * @param pTasks SArray**<STaskInfo>
+ * @return
+ */
+int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks);
+
+void schedulerFreeTaskList(SArray *taskList);
+
+
 #ifdef __cplusplus
 }
 #endif

include/util/taoserror.h

@@ -307,6 +307,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_VND_NO_WRITE_AUTH             TAOS_DEF_ERROR_CODE(0, 0x0512)  //"Database write operation denied")
 #define TSDB_CODE_VND_IS_SYNCING                TAOS_DEF_ERROR_CODE(0, 0x0513)  //"Database is syncing")
 #define TSDB_CODE_VND_INVALID_TSDB_STATE        TAOS_DEF_ERROR_CODE(0, 0x0514)  //"Invalid tsdb state")
+#define TSDB_CODE_VND_TB_NOT_EXIST              TAOS_DEF_ERROR_CODE(0, 0x0515)  // "Table not exists")
 
 // tsdb
 #define TSDB_CODE_TDB_INVALID_TABLE_ID          TAOS_DEF_ERROR_CODE(0, 0x0600)  //"Invalid table ID")
@@ -355,7 +356,11 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_QRY_TASK_ALREADY_EXIST        TAOS_DEF_ERROR_CODE(0, 0x0712)  //"Task already exist")
 #define TSDB_CODE_QRY_RES_CACHE_NOT_EXIST       TAOS_DEF_ERROR_CODE(0, 0x0713)  //"Task result cache not exist")
 #define TSDB_CODE_QRY_TASK_CANCELLED            TAOS_DEF_ERROR_CODE(0, 0x0714)  //"Task cancelled")
-
+#define TSDB_CODE_QRY_TASK_DROPPED              TAOS_DEF_ERROR_CODE(0, 0x0715)  //"Task dropped")
+#define TSDB_CODE_QRY_TASK_CANCELLING           TAOS_DEF_ERROR_CODE(0, 0x0716)  //"Task cancelling")
+#define TSDB_CODE_QRY_TASK_DROPPING             TAOS_DEF_ERROR_CODE(0, 0x0717)  //"Task dropping")
+#define TSDB_CODE_QRY_DUPLICATTED_OPERATION     TAOS_DEF_ERROR_CODE(0, 0x0718)  //"Duplicatted operation")
+#define TSDB_CODE_QRY_TASK_MSG_ERROR            TAOS_DEF_ERROR_CODE(0, 0x0719)  //"Task message error")
 
 // grant
 #define TSDB_CODE_GRANT_EXPIRED                 TAOS_DEF_ERROR_CODE(0, 0x0800)  //"License expired")

include/util/tcoding.h

@@ -351,6 +351,7 @@ static FORCE_INLINE void *taosDecodeString(void *buf, char **value) {
 
   buf = taosDecodeVariantU64(buf, &size);
   *value = (char *)malloc((size_t)size + 1);
+
   if (*value == NULL) return NULL;
   memcpy(*value, buf, (size_t)size);
 

include/util/tdef.h

@@ -181,7 +181,7 @@ do { \
 #define TSDB_COL_NAME_LEN         65
 #define TSDB_MAX_SAVED_SQL_LEN    TSDB_MAX_COLUMNS * 64
 #define TSDB_MAX_SQL_LEN          TSDB_PAYLOAD_SIZE
-#define TSDB_MAX_SQL_SHOW_LEN     512
+#define TSDB_MAX_SQL_SHOW_LEN     1024
 #define TSDB_MAX_ALLOWED_SQL_LEN  (1*1024*1024u)          // sql length should be less than 1mb
 
 #define TSDB_APP_NAME_LEN          TSDB_UNI_LEN

source/client/inc/clientHb.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "os.h"
-#include "tarray.h"
-#include "thash.h"
-#include "tmsg.h"
-
-#define HEARTBEAT_INTERVAL 1500  // ms
-
-typedef enum {
-  HEARTBEAT_TYPE_MQ = 0,
-  // types can be added here
-  //
-  HEARTBEAT_TYPE_MAX
-} EHbType;
-
-typedef int32_t (*FHbRspHandle)(SClientHbRsp* pReq);
-
-typedef struct SAppHbMgr {
-  // statistics
-  int32_t reportCnt;
-  int32_t connKeyCnt;
-  int64_t reportBytes;  // not implemented
-  int64_t startTime;
-  // ctl
-  SRWLatch lock;  // lock is used in serialization
-  // connection
-  void*  transporter;
-  SEpSet epSet;
-  // info
-  SHashObj* activeInfo;    // hash<SClientHbKey, SClientHbReq>
-  SHashObj* getInfoFuncs;  // hash<SClientHbKey, FGetConnInfo>
-} SAppHbMgr;
-
-typedef struct SClientHbMgr {
-  int8_t inited;
-  // ctl
-  int8_t          threadStop;
-  pthread_t       thread;
-  pthread_mutex_t lock;       // used when app init and cleanup
-  SArray*         appHbMgrs;  // SArray<SAppHbMgr*> one for each cluster
-  FHbRspHandle    handle[HEARTBEAT_TYPE_MAX];
-} SClientHbMgr;
-
-// TODO: embed param into function
-// return type: SArray<Skv>
-typedef SArray* (*FGetConnInfo)(SClientHbKey connKey, void* param);
-
-// global, called by mgmt
-int  hbMgrInit();
-void hbMgrCleanUp();
-int  hbHandleRsp(SClientHbBatchRsp* hbRsp);
-
-// cluster level
-SAppHbMgr* appHbMgrInit(void* transporter, SEpSet epSet);
-void appHbMgrCleanup(SAppHbMgr* pAppHbMgr);
-
-// conn level
-int  hbRegisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, FGetConnInfo func);
-void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey);
-
-int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen);
-
-// mq
-void hbMgrInitMqHbRspHandle();

source/client/inc/clientInt.h

@@ -31,6 +31,41 @@ extern "C" {
 #include "trpc.h"
 #include "query.h"
 
+#define HEARTBEAT_INTERVAL 1500  // ms
+
+typedef struct SAppInstInfo SAppInstInfo;
+
+typedef int32_t (*FHbRspHandle)(SClientHbRsp* pReq);
+
+typedef struct SAppHbMgr {
+  // statistics
+  int32_t reportCnt;
+  int32_t connKeyCnt;
+  int64_t reportBytes;  // not implemented
+  int64_t startTime;
+  // ctl
+  SRWLatch lock;  // lock is used in serialization
+  // connection
+  SAppInstInfo* pAppInstInfo;
+  // info
+  SHashObj* activeInfo;    // hash<SClientHbKey, SClientHbReq>
+  SHashObj* getInfoFuncs;  // hash<SClientHbKey, FGetConnInfo>
+} SAppHbMgr;
+
+typedef struct SClientHbMgr {
+  int8_t inited;
+  // ctl
+  int8_t          threadStop;
+  pthread_t       thread;
+  pthread_mutex_t lock;       // used when app init and cleanup
+  SArray*         appHbMgrs;  // SArray<SAppHbMgr*> one for each cluster
+  FHbRspHandle    handle[HEARTBEAT_TYPE_MAX];
+} SClientHbMgr;
+
+// TODO: embed param into function
+// return type: SArray<Skv>
+typedef SArray* (*FGetConnInfo)(SClientHbKey connKey, void* param);
+
 typedef struct SQueryExecMetric {
   int64_t      start;    // start timestamp
   int64_t      parsed;   // start to parse
@@ -55,15 +90,15 @@ typedef struct SHeartBeatInfo {
   void  *pTimer;   // timer, used to send request msg to mnode
 } SHeartBeatInfo;
 
-typedef struct SAppInstInfo {
+struct SAppInstInfo {
   int64_t          numOfConns;
   SCorEpSet        mgmtEp;
   SInstanceSummary summary;
   SList            *pConnList;  // STscObj linked list
   int64_t          clusterId;
   void             *pTransporter;
-  SHeartBeatInfo hb;
-} SAppInstInfo;
+  struct SAppHbMgr *pAppHbMgr;
+};
 
 typedef struct SAppInfo {
   int64_t        startTime;
@@ -81,13 +116,17 @@ typedef struct STscObj {
   char             db[TSDB_DB_FNAME_LEN];
   int32_t          acctId;
   uint32_t         connId;
+  int32_t          connType;
   uint64_t         id;       // ref ID returned by taosAddRef
-  void            *pTransporter;
   pthread_mutex_t  mutex;     // used to protect the operation on db
-  int32_t          numOfReqs; // number of sqlObj from this tscObj
+  int32_t          numOfReqs; // number of sqlObj bound to this connection
   SAppInstInfo    *pAppInfo;
 } STscObj;
 
+typedef struct SMqConsumer {
+  STscObj* pTscObj;
+} SMqConsumer;
+
 typedef struct SReqResultInfo {
   const char  *pRspMsg;
   const char  *pData;
@@ -112,8 +151,9 @@ typedef struct SRequestSendRecvBody {
   tsem_t            rspSem;        // not used now
   void*             fp;
   SShowReqInfo      showInfo;      // todo this attribute will be removed after the query framework being completed.
-  struct SSchJob *pQueryJob;     // query job, created according to sql query DAG.
   SDataBuf          requestMsg;
+  struct SSchJob   *pQueryJob;     // query job, created according to sql query DAG.
+  struct SQueryDag *pDag;          // the query dag, generated according to the sql statement.
   SReqResultInfo    resInfo;
 } SRequestSendRecvBody;
 
@@ -169,6 +209,26 @@ void *doFetchRow(SRequestObj* pRequest);
 
 void  setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows);
 
+// --- heartbeat 
+// global, called by mgmt
+int  hbMgrInit();
+void hbMgrCleanUp();
+int  hbHandleRsp(SClientHbBatchRsp* hbRsp);
+
+// cluster level
+SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo);
+void appHbMgrCleanup(SAppHbMgr* pAppHbMgr);
+
+// conn level
+int  hbRegisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, FGetConnInfo func);
+void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey);
+
+int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen);
+
+// --- mq
+void hbMgrInitMqHbRspHandle();
+
+
 #ifdef __cplusplus
 }
 #endif

source/client/src/clientEnv.c

@@ -89,13 +89,12 @@ static void tscInitLogFile() {
 
 // todo close the transporter properly
 void closeTransporter(STscObj* pTscObj)  {
-  if (pTscObj == NULL || pTscObj->pTransporter == NULL) {
+  if (pTscObj == NULL || pTscObj->pAppInfo->pTransporter == NULL) {
     return;
   }
 
-  tscDebug("free transporter:%p in connObj: 0x%"PRIx64, pTscObj->pTransporter, pTscObj->id);
-  rpcClose(pTscObj->pTransporter);
-  pTscObj->pTransporter = NULL;
+  tscDebug("free transporter:%p in connObj: 0x%"PRIx64, pTscObj->pAppInfo->pTransporter, pTscObj->id);
+  rpcClose(pTscObj->pAppInfo->pTransporter);
 }
 
 // TODO refactor
@@ -140,10 +139,6 @@ void* createTscObj(const char* user, const char* auth, const char *db, SAppInstI
   }
 
   pObj->pAppInfo = pAppInfo;
-  if (pAppInfo != NULL) {
-    pObj->pTransporter = pAppInfo->pTransporter;
-  }
-
   tstrncpy(pObj->user, user, sizeof(pObj->user));
   memcpy(pObj->pass, auth, TSDB_PASSWORD_LEN);
 
@@ -199,6 +194,7 @@ static void doDestroyRequest(void* p) {
   tfree(pRequest->pInfo);
 
   doFreeReqResultInfo(&pRequest->body.resInfo);
+  qDestroyQueryDag(pRequest->body.pDag);
 
   deregisterRequest(pRequest);
   tfree(pRequest);

source/client/src/clientHb.c

@@ -13,7 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "clientHb.h"
+#include "clientInt.h"
 #include "trpc.h"
 
 static SClientHbMgr clientHbMgr = {0};
@@ -21,10 +21,18 @@ static SClientHbMgr clientHbMgr = {0};
 static int32_t hbCreateThread();
 static void    hbStopThread();
 
-static int32_t hbMqHbRspHandle(SClientHbRsp* pReq) {
+static int32_t hbMqHbRspHandle(SClientHbRsp* pRsp) {
   return 0;
 }
 
+static int32_t hbMqAsyncCallBack(void* param, const SDataBuf* pMsg, int32_t code) {
+  if (code != 0) {
+    return -1;
+  }
+  SClientHbRsp* pRsp = (SClientHbRsp*) pMsg->pData;
+  return hbMqHbRspHandle(pRsp);
+}
+
 void hbMgrInitMqHbRspHandle() {
   clientHbMgr.handle[HEARTBEAT_TYPE_MQ] = hbMqHbRspHandle;
 }
@@ -35,18 +43,18 @@ static FORCE_INLINE void hbMgrInitHandle() {
 }
 
 SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) {
-  SClientHbBatchReq* pReq = malloc(sizeof(SClientHbBatchReq));
-  if (pReq == NULL) {
+  SClientHbBatchReq* pBatchReq = malloc(sizeof(SClientHbBatchReq));
+  if (pBatchReq == NULL) {
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return NULL;
   }
   int32_t connKeyCnt = atomic_load_32(&pAppHbMgr->connKeyCnt);
-  pReq->reqs = taosArrayInit(connKeyCnt, sizeof(SClientHbReq));
+  pBatchReq->reqs = taosArrayInit(connKeyCnt, sizeof(SClientHbReq));
 
   void *pIter = taosHashIterate(pAppHbMgr->activeInfo, NULL);
   while (pIter != NULL) {
-    taosArrayPush(pReq->reqs, pIter);
     SClientHbReq* pOneReq = pIter;
+    taosArrayPush(pBatchReq->reqs, pOneReq);
     taosHashClear(pOneReq->info);
 
     pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter);
@@ -59,10 +67,10 @@ SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) {
     taosHashCopyKey(pIter, &connKey);
     getConnInfoFp(connKey, NULL);
 
-    pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter);
+    pIter = taosHashIterate(pAppHbMgr->getInfoFuncs, pIter);
   }
 
-  return pReq;
+  return pBatchReq;
 }
 
 static void* hbThreadFunc(void* param) {
@@ -75,20 +83,48 @@ static void* hbThreadFunc(void* param) {
 
     int sz = taosArrayGetSize(clientHbMgr.appHbMgrs);
     for(int i = 0; i < sz; i++) {
-      SAppHbMgr* pAppHbMgr = taosArrayGet(clientHbMgr.appHbMgrs, i);
-      SClientHbBatchReq* pReq = hbGatherAllInfo(pAppHbMgr);
-      void* reqStr = NULL;
-      int tlen = tSerializeSClientHbBatchReq(&reqStr, pReq);
-      SMsgSendInfo info;
-      /*info.fp = hbHandleRsp;*/
+      SAppHbMgr* pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i);
 
+      int32_t connCnt = atomic_load_32(&pAppHbMgr->connKeyCnt);
+      if (connCnt == 0) {
+        continue;
+      }
+      SClientHbBatchReq* pReq = hbGatherAllInfo(pAppHbMgr);
+      if (pReq == NULL) {
+        continue;
+      }
+      int tlen = tSerializeSClientHbBatchReq(NULL, pReq);
+      void *buf = malloc(tlen);
+      if (buf == NULL) {
+        //TODO: error handling
+        break;
+      }
+      void *bufCopy = buf;
+      tSerializeSClientHbBatchReq(&bufCopy, pReq);
+      SMsgSendInfo *pInfo = malloc(sizeof(SMsgSendInfo));
+      if (pInfo == NULL) {
+        terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+        tFreeClientHbBatchReq(pReq);
+        free(buf);
+        break;
+      }
+      pInfo->fp = hbMqAsyncCallBack;
+      pInfo->msgInfo.pData = buf;
+      pInfo->msgInfo.len = tlen;
+      pInfo->msgType = TDMT_MND_HEARTBEAT;
+      pInfo->param = NULL;
+      pInfo->requestId = generateRequestId();
+      pInfo->requestObjRefId = 0;
+
+      SAppInstInfo *pAppInstInfo = pAppHbMgr->pAppInstInfo;
       int64_t transporterId = 0;
-      asyncSendMsgToServer(pAppHbMgr->transporter, &pAppHbMgr->epSet, &transporterId, &info);
+      SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp);
+      asyncSendMsgToServer(pAppInstInfo->pTransporter, &epSet, &transporterId, pInfo);
       tFreeClientHbBatchReq(pReq);
 
       atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1);
-      taosMsleep(HEARTBEAT_INTERVAL);
     }
+    taosMsleep(HEARTBEAT_INTERVAL);
   }
   return NULL;
 }
@@ -110,7 +146,8 @@ static void hbStopThread() {
   atomic_store_8(&clientHbMgr.threadStop, 1);
 }
 
-SAppHbMgr* appHbMgrInit(void* transporter, SEpSet epSet) {
+SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo) {
+  hbMgrInit();
   SAppHbMgr* pAppHbMgr = malloc(sizeof(SAppHbMgr)); 
   if (pAppHbMgr == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -119,16 +156,27 @@ SAppHbMgr* appHbMgrInit(void* transporter, SEpSet epSet) {
   // init stat
   pAppHbMgr->startTime = taosGetTimestampMs();
 
-  // init connection info
-  pAppHbMgr->transporter = transporter;
-  pAppHbMgr->epSet = epSet;
+  // init app info
+  pAppHbMgr->pAppInstInfo = pAppInstInfo;
 
   // init hash info
   pAppHbMgr->activeInfo = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
+
+  if (pAppHbMgr->activeInfo == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    free(pAppHbMgr);
+    return NULL;
+  }
   pAppHbMgr->activeInfo->freeFp = tFreeClientHbReq;
   // init getInfoFunc
   pAppHbMgr->getInfoFuncs = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
 
+  if (pAppHbMgr->getInfoFuncs == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    free(pAppHbMgr);
+    return NULL;
+  }
+
   taosArrayPush(clientHbMgr.appHbMgrs, &pAppHbMgr);
   return pAppHbMgr;
 }
@@ -138,7 +186,7 @@ void appHbMgrCleanup(SAppHbMgr* pAppHbMgr) {
 
   int sz = taosArrayGetSize(clientHbMgr.appHbMgrs);
   for (int i = 0; i < sz; i++) {
-    SAppHbMgr* pTarget = taosArrayGet(clientHbMgr.appHbMgrs, i);
+    SAppHbMgr* pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i);
     if (pAppHbMgr == pTarget) {
       taosHashCleanup(pTarget->activeInfo);
       taosHashCleanup(pTarget->getInfoFuncs);
@@ -171,7 +219,6 @@ void hbMgrCleanUp() {
   if (old == 0) return;
 
   taosArrayDestroy(clientHbMgr.appHbMgrs);
-
 }
 
 int hbHandleRsp(SClientHbBatchRsp* hbRsp) {

source/client/src/clientImpl.c

@@ -13,12 +13,12 @@
 #include "tpagedfile.h"
 #include "tref.h"
 
-#define CHECK_CODE_GOTO(expr, lable) \
+#define CHECK_CODE_GOTO(expr, label) \
   do {                               \
     int32_t code = expr;             \
     if (TSDB_CODE_SUCCESS != code) { \
       terrno = code;                 \
-      goto lable;                    \
+      goto label;                    \
     }                                \
   } while (0)
 
@@ -58,7 +58,7 @@ static char* getClusterKey(const char* user, const char* auth, const char* ip, i
   return strdup(key);
 }
 
-static STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo);
+static STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo);
 static void  setResSchemaInfo(SReqResultInfo* pResInfo, const SDataBlockSchema* pDataBlockSchema);
 
 TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) {
@@ -71,18 +71,18 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass,
     return NULL;
   }
 
-  char tmp[TSDB_DB_NAME_LEN] = {0};
+  char localDb[TSDB_DB_NAME_LEN] = {0};
   if (db != NULL) {
     if(!validateDbName(db)) {
       terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH;
       return NULL;
     }
 
-    tstrncpy(tmp, db, sizeof(tmp));
-    strdequote(tmp);
+    tstrncpy(localDb, db, sizeof(localDb));
+    strdequote(localDb);
   }
 
-  char secretEncrypt[32] = {0};
+  char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0};
   if (auth == NULL) {
     if (!validatePassword(pass)) {
       terrno = TSDB_CODE_TSC_INVALID_PASS_LENGTH;
@@ -111,18 +111,20 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass,
 
   char* key = getClusterKey(user, secretEncrypt, ip, port);
 
+  // TODO: race condition here.
   SAppInstInfo** pInst = taosHashGet(appInfo.pInstMap, key, strlen(key));
   if (pInst == NULL) {
     SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo));
     p->mgmtEp       = epSet;
     p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores);
+    p->pAppHbMgr = appHbMgrInit(p);
     taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES);
 
     pInst = &p;
   }
 
   tfree(key);
-  return taosConnectImpl(ip, user, &secretEncrypt[0], db, port, NULL, NULL, *pInst);
+  return taosConnectImpl(user, &secretEncrypt[0], localDb, port, NULL, NULL, *pInst);
 }
 
 int32_t buildRequest(STscObj *pTscObj, const char *sql, int sqlLen, SRequestObj** pRequest) {
@@ -154,11 +156,11 @@ int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery) {
     .requestId = pRequest->requestId,
     .acctId    = pTscObj->acctId,
     .db        = getConnectionDB(pTscObj),
-    .pTransporter = pTscObj->pTransporter,
     .pSql      = pRequest->sqlstr,
     .sqlLen    = pRequest->sqlLen,
     .pMsg      = pRequest->msgBuf,
-    .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE
+    .msgLen    = ERROR_MSG_BUF_DEFAULT_SIZE,
+    .pTransporter = pTscObj->pAppInfo->pTransporter,
   };
 
   cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
@@ -191,10 +193,10 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) {
         pShowReqInfo->pArray = pDcl->pExtension;
       }
     }
-    asyncSendMsgToServer(pTscObj->pTransporter, &pDcl->epSet, &transporterId, pSendMsg);
+    asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pDcl->epSet, &transporterId, pSendMsg);
   } else {
     SEpSet* pEpSet = &pTscObj->pAppInfo->mgmtEp.epSet;
-    asyncSendMsgToServer(pTscObj->pTransporter, pEpSet, &transporterId, pSendMsg);
+    asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, pEpSet, &transporterId, pSendMsg);
   }
 
   tsem_wait(&pRequest->body.rspSem);
@@ -241,7 +243,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) {
   if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) {
     SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf};
 
-    int32_t code = scheduleExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, &pRequest->body.pQueryJob, &res);
+    int32_t code = scheduleExecJob(pRequest->pTscObj->pAppInfo->pTransporter, NULL, pDag, &pRequest->body.pQueryJob, &res);
     if (code != TSDB_CODE_SUCCESS) {
       // handle error and retry
     } else {
@@ -255,39 +257,163 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) {
     return pRequest->code;
   }
 
-  return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, &pRequest->body.pQueryJob);
+  return scheduleAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, NULL, pDag, &pRequest->body.pQueryJob);
 }
 
-TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen) {
-  STscObj* pTscObj = (STscObj*)taos;
-  SRequestObj* pRequest = NULL;
-  SQueryNode*  pQuery = NULL;
-  SQueryDag*   pDag = NULL;
-  char *dagStr = NULL;
+typedef struct tmq_t tmq_t;
+
+typedef struct SMqClientTopic {
+  // subscribe info
+  int32_t sqlLen;
+  char*   sql;
+  char*   topicName;
+  int64_t topicId;
+  // statistics
+  int64_t consumeCnt;
+  // offset
+  int64_t committedOffset;
+  int64_t currentOffset;
+  //connection info
+  int32_t vgId;
+  SEpSet  epSet;
+} SMqClientTopic;
+
+typedef struct tmq_resp_err_t {
+  int32_t code;
+} tmq_resp_err_t;
+
+typedef struct tmq_topic_vgroup_list_t {
+  char* topicName;
+  int32_t vgId;
+  int64_t committedOffset;
+} tmq_topic_vgroup_list_t;
+
+typedef void (tmq_commit_cb(tmq_t*, tmq_resp_err_t, tmq_topic_vgroup_list_t*, void* param));
+
+typedef struct tmq_conf_t{
+  char*          clientId;
+  char*          groupId;
+  char*          ip;
+  uint16_t       port;
+  tmq_commit_cb* commit_cb;
+} tmq_conf_t;
+
+struct tmq_t {
+  char           groupId[256];
+  char           clientId[256];
+  STscObj*       pTscObj;
+  tmq_commit_cb* commit_cb;
+  SArray*        clientTopics;  // SArray<SMqClientTopic>
+};
+
+void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) {
+  conf->commit_cb = cb;
+}
+
+SArray* tmqGetConnInfo(SClientHbKey connKey, void* param) {
+  tmq_t* pTmq = (void*)param;
+  SArray* pArray = taosArrayInit(0, sizeof(SKv));
+  if (pArray == NULL) {
+    return NULL;
+  }
+  SKv kv = {0};
+  kv.key = malloc(256);
+  if (kv.key == NULL) {
+    taosArrayDestroy(pArray);
+    return NULL;
+  }
+  strcpy(kv.key, "mq-tmp");
+  kv.keyLen = strlen("mq-tmp") + 1;
+  SMqHbMsg* pMqHb = malloc(sizeof(SMqHbMsg));
+  if (pMqHb == NULL) {
+    return pArray;
+  }
+  pMqHb->consumerId = connKey.connId;
+  SArray* clientTopics = pTmq->clientTopics;
+  int sz = taosArrayGetSize(clientTopics);
+  for (int i = 0; i < sz; i++) {
+    SMqClientTopic* pCTopic = taosArrayGet(clientTopics, i);
+    if (pCTopic->vgId == -1) {
+      pMqHb->status = 1;
+      break;
+    }
+  }
+  kv.value = pMqHb;
+  kv.valueLen = sizeof(SMqHbMsg);
+  taosArrayPush(pArray, &kv);
+
+  return pArray;
+}
+
+tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) {
+  tmq_t* pTmq = malloc(sizeof(tmq_t));
+  if (pTmq == NULL) {
+    return NULL;
+  }
+  strcpy(pTmq->groupId, conf->groupId);
+  strcpy(pTmq->clientId, conf->clientId);
+  pTmq->pTscObj = (STscObj*)conn;
+  pTmq->pTscObj->connType = HEARTBEAT_TYPE_MQ;
+
+  return pTmq;
+}
+
+TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) {
+  STscObj     *pTscObj = (STscObj*)taos;
+  SRequestObj *pRequest = NULL;
+  SQueryNode  *pQueryNode = NULL;
+  char        *pStr = NULL;
 
   terrno = TSDB_CODE_SUCCESS;
+  if (taos == NULL || topicName == NULL || sql == NULL) {
+    tscError("invalid parameters for creating topic, connObj:%p, topic name:%s, sql:%s", taos, topicName, sql);
+    terrno = TSDB_CODE_TSC_INVALID_INPUT;
+    goto _return;
+  }
+
+  if (strlen(topicName) >= TSDB_TOPIC_NAME_LEN) {
+    tscError("topic name too long, max length:%d", TSDB_TOPIC_NAME_LEN - 1);
+    terrno = TSDB_CODE_TSC_INVALID_INPUT;
+    goto _return;
+  }
+
+  if (sqlLen > tsMaxSQLStringLen) {
+    tscError("sql string exceeds max length:%d", tsMaxSQLStringLen);
+    terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
+    goto _return;
+  }
+
+  tscDebug("start to create topic, %s", topicName);
 
   CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
+  CHECK_CODE_GOTO(parseSql(pRequest, &pQueryNode), _return);
 
-//temporary disabled until planner ready
-#if 0
-  CHECK_CODE_GOTO(parseSql(pRequest, &pQuery), _return);
-  //TODO: check sql valid
+  // todo check for invalid sql statement and return with error code
 
-  CHECK_CODE_GOTO(qCreateQueryDag(pQuery, &pDag), _return);
+  CHECK_CODE_GOTO(qCreateQueryDag(pQueryNode, &pRequest->body.pDag, pRequest->requestId), _return);
 
-  dagStr = qDagToString(pDag);
-  if(dagStr == NULL) {
-    //TODO
+  pStr = qDagToString(pRequest->body.pDag);
+  if(pStr == NULL) {
+    goto _return;
   }
-#endif
+
+  // The topic should be related to a database that the queried table is belonged to.
+  SName name = {0};
+  char dbName[TSDB_DB_FNAME_LEN] = {0};
+  tNameGetFullDbName(&((SQueryStmtInfo*) pQueryNode)->pTableMetaInfo[0]->name, dbName);
+
+  tNameFromString(&name, dbName, T_NAME_ACCT|T_NAME_DB);
+  tNameFromString(&name, topicName, T_NAME_TABLE);
+
+  char topicFname[TSDB_TOPIC_FNAME_LEN] = {0};
+  tNameExtractFullName(&name, topicFname);
 
   SCMCreateTopicReq req = {
-    .name = (char*)name,
+    .name         = (char*) topicFname,
     .igExists     = 0,
-    /*.physicalPlan = dagStr,*/
-    .physicalPlan = (char*)sql,
-    .logicalPlan = "",
+    .physicalPlan = (char*) pStr,
+    .sql          = (char*) sql,
+    .logicalPlan  = "no logic plan",
   };
 
   int tlen = tSerializeSCMCreateTopicReq(NULL, &req);
@@ -295,30 +421,54 @@ TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sq
   if(buf == NULL) {
     goto _return;
   }
+
   void* abuf = buf;
   tSerializeSCMCreateTopicReq(&abuf, &req);
   /*printf("formatted: %s\n", dagStr);*/
 
   pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen };
+  pRequest->type = TDMT_MND_CREATE_TOPIC;
 
   SMsgSendInfo* body = buildMsgInfoImpl(pRequest);
-  SEpSet* pEpSet = &pTscObj->pAppInfo->mgmtEp.epSet;
+  SEpSet epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
 
   int64_t transporterId = 0;
-  asyncSendMsgToServer(pTscObj->pTransporter, pEpSet, &transporterId, body);
+  asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, body);
 
   tsem_wait(&pRequest->body.rspSem);
 
 _return:
-  qDestroyQuery(pQuery);
-  qDestroyQueryDag(pDag); 
+  qDestroyQuery(pQueryNode);
+  if (body != NULL) {
     destroySendMsgInfo(body);
+  }
+
   if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {
     pRequest->code = terrno;
   }
+
   return pRequest;
 }
 
+typedef struct tmq_message_t {
+  int32_t  numOfRows;
+  char*    topicName;
+  TAOS_ROW row[];
+} tmq_message_t;
+
+tmq_message_t* tmq_consume_poll(tmq_t* mq, int64_t blocking_time) {
+  return NULL;
+}
+
+tmq_resp_err_t* tmq_commit(tmq_t* mq, void* callback, int32_t async) {
+  return NULL;
+}
+
+void tmq_message_destroy(tmq_message_t* mq_message) {
+
+}
+
+
 TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) {
   STscObj *pTscObj = (STscObj *)taos;
   if (sqlLen > (size_t) tsMaxSQLStringLen) {
@@ -330,24 +480,22 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) {
   nPrintTsc("%s", sql)
 
   SRequestObj *pRequest = NULL;
-  SQueryNode  *pQuery   = NULL;
-  SQueryDag   *pDag     = NULL;
+  SQueryNode  *pQueryNode = NULL;
 
   terrno = TSDB_CODE_SUCCESS;
   CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
-  CHECK_CODE_GOTO(parseSql(pRequest, &pQuery), _return);
+  CHECK_CODE_GOTO(parseSql(pRequest, &pQueryNode), _return);
 
-  if (qIsDdlQuery(pQuery)) {
-    CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return);
+  if (qIsDdlQuery(pQueryNode)) {
+    CHECK_CODE_GOTO(execDdlQuery(pRequest, pQueryNode), _return);
   } else {
-    CHECK_CODE_GOTO(getPlan(pRequest, pQuery, &pDag), _return);
-    CHECK_CODE_GOTO(scheduleQuery(pRequest, pDag), _return);
+    CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag), _return);
+    CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag), _return);
     pRequest->code = terrno;
   }
 
 _return:
-  qDestroyQuery(pQuery);
-  qDestroyQueryDag(pDag);
+  qDestroyQuery(pQueryNode);
   if (NULL != pRequest && TSDB_CODE_SUCCESS != terrno) {
     pRequest->code = terrno;
   }
@@ -391,7 +539,7 @@ int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSe
   return 0;
 }
 
-STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo) {
+STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo) {
   STscObj *pTscObj = createTscObj(user, auth, db, pAppInfo);
   if (NULL == pTscObj) {
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
@@ -408,7 +556,7 @@ STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, con
   SMsgSendInfo* body = buildConnectMsg(pRequest);
 
   int64_t transporterId = 0;
-  asyncSendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body);
+  asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body);
 
   tsem_wait(&pRequest->body.rspSem);
   if (pRequest->code != TSDB_CODE_SUCCESS) {
@@ -419,7 +567,7 @@ STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, con
     taos_close(pTscObj);
     pTscObj = NULL;
   } else {
-    tscDebug("0x%"PRIx64" connection is opening, connId:%d, dnodeConn:%p, reqId:0x%"PRIx64, pTscObj->id, pTscObj->connId, pTscObj->pTransporter, pRequest->requestId);
+    tscDebug("0x%"PRIx64" connection is opening, connId:%d, dnodeConn:%p, reqId:0x%"PRIx64, pTscObj->id, pTscObj->connId, pTscObj->pAppInfo->pTransporter, pRequest->requestId);
     destroyRequest(pRequest);
   }
 
@@ -450,7 +598,9 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest) {
   STscObj *pObj = pRequest->pTscObj;
 
   char* db = getConnectionDB(pObj);
+  if (db != NULL) {
     tstrncpy(pConnect->db, db, sizeof(pConnect->db));
+  }
   tfree(db);
 
   pConnect->pid = htonl(appInfo.pid);
@@ -587,7 +737,7 @@ void* doFetchRow(SRequestObj* pRequest) {
 
       int64_t  transporterId = 0;
       STscObj *pTscObj = pRequest->pTscObj;
-      asyncSendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body);
+      asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body);
       tsem_wait(&pRequest->body.rspSem);
 
       pRequest->type = TDMT_VND_SHOW_TABLES_FETCH;
@@ -597,7 +747,7 @@ void* doFetchRow(SRequestObj* pRequest) {
 
     int64_t  transporterId = 0;
     STscObj *pTscObj = pRequest->pTscObj;
-    asyncSendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body);
+    asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body);
 
     tsem_wait(&pRequest->body.rspSem);
 
@@ -650,9 +800,12 @@ void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t
 char* getConnectionDB(STscObj* pObj) {
   char *p = NULL;
   pthread_mutex_lock(&pObj->mutex);
+  size_t len = strlen(pObj->db);
+  if (len > 0) {
     p = strndup(pObj->db, tListLen(pObj->db));
-  pthread_mutex_unlock(&pObj->mutex);
+  }
 
+  pthread_mutex_unlock(&pObj->mutex);
   return p;
 }
 

source/client/src/clientMsgHandler.c

@@ -71,6 +71,9 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
   pTscObj->pAppInfo->clusterId = pConnect->clusterId;
   atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
 
+  SClientHbKey connKey = {.connId = pConnect->connId, .hbType = HEARTBEAT_TYPE_QUERY};
+  hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey, NULL);
+
   //  pRequest->body.resInfo.pRspMsg = pMsg->pData;
   tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, pConnect->clusterId,
            pTscObj->pAppInfo->numOfConns);

source/client/test/clientTests.cpp

@@ -49,10 +49,11 @@ int main(int argc, char** argv) {
 TEST(testCase, driverInit_Test) { taos_init(); }
 
 TEST(testCase, connect_Test) {
-  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  TAOS* pConn = taos_connect("localhost", "root", "taosdata", "abc1", 0);
   if (pConn == NULL) {
     printf("failed to connect to server, reason:%s\n", taos_errstr(NULL));
   }
+  sleep(3);
   taos_close(pConn);
 }
 
@@ -148,27 +149,27 @@ TEST(testCase, connect_Test) {
 //}
 //
 //TEST(testCase, create_db_Test) {
-//  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
-//  assert(pConn != NULL);
-//
-//  TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2");
-//  if (taos_errno(pRes) != 0) {
-//    printf("error in create db, reason:%s\n", taos_errstr(pRes));
-//  }
-//
-//  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
-//  ASSERT_TRUE(pFields == NULL);
-//
-//  int32_t numOfFields = taos_num_fields(pRes);
-//  ASSERT_EQ(numOfFields, 0);
-//
-//  taos_free_result(pRes);
-//
-//  pRes = taos_query(pConn, "create database abc1 vgroups 4");
-//  if (taos_errno(pRes) != 0) {
-//    printf("error in create db, reason:%s\n", taos_errstr(pRes));
-//  }
-//  taos_close(pConn);
+  //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  //assert(pConn != NULL);
+
+  //TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2");
+  //if (taos_errno(pRes) != 0) {
+    //printf("error in create db, reason:%s\n", taos_errstr(pRes));
+  //}
+
+  //TAOS_FIELD* pFields = taos_fetch_fields(pRes);
+  //ASSERT_TRUE(pFields == NULL);
+
+  //int32_t numOfFields = taos_num_fields(pRes);
+  //ASSERT_EQ(numOfFields, 0);
+
+  //taos_free_result(pRes);
+
+  //pRes = taos_query(pConn, "create database abc1 vgroups 4");
+  //if (taos_errno(pRes) != 0) {
+    //printf("error in create db, reason:%s\n", taos_errstr(pRes));
+  //}
+  //taos_close(pConn);
 //}
 //
 //TEST(testCase, create_dnode_Test) {
@@ -194,7 +195,7 @@ TEST(testCase, connect_Test) {
 //  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
 //  assert(pConn != NULL);
 //
-//  TAOS_RES* pRes = taos_query(pConn, "drop dnode 2");
+//  TAOS_RES* pRes = taos_query(pConn, "drop dnode 3");
 //  if (taos_errno(pRes) != 0) {
 //    printf("error in drop dnode, reason:%s\n", taos_errstr(pRes));
 //  }
@@ -205,6 +206,11 @@ TEST(testCase, connect_Test) {
 //  int32_t numOfFields = taos_num_fields(pRes);
 //  ASSERT_EQ(numOfFields, 0);
 //
+//  pRes = taos_query(pConn, "drop dnode 4");
+//  if (taos_errno(pRes) != 0) {
+//    printf("error in drop dnode, reason:%s\n", taos_errstr(pRes));
+//  }
+//
 //  taos_free_result(pRes);
 //  taos_close(pConn);
 //}
@@ -227,45 +233,39 @@ TEST(testCase, connect_Test) {
 //  taos_close(pConn);
 //}
 //
-//// TEST(testCase, drop_db_test) {
-////  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
-////  assert(pConn != NULL);
-////
-////  showDB(pConn);
-////
-////  TAOS_RES* pRes = taos_query(pConn, "drop database abc1");
-////  if (taos_errno(pRes) != 0) {
-////    printf("failed to drop db, reason:%s\n", taos_errstr(pRes));
-////  }
-////  taos_free_result(pRes);
-////
-////  showDB(pConn);
-////
-////  pRes = taos_query(pConn, "create database abc1");
-////  if (taos_errno(pRes) != 0) {
-////    printf("create to drop db, reason:%s\n", taos_errstr(pRes));
-////  }
-////  taos_free_result(pRes);
-////  taos_close(pConn);
-////}
-//
-//TEST(testCase, create_stable_Test) {
+// TEST(testCase, drop_db_test) {
 //  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
 //  assert(pConn != NULL);
 //
-//  TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2");
+//  showDB(pConn);
+//
+//  TAOS_RES* pRes = taos_query(pConn, "drop database abc1");
 //  if (taos_errno(pRes) != 0) {
-//    printf("error in create db, reason:%s\n", taos_errstr(pRes));
+//    printf("failed to drop db, reason:%s\n", taos_errstr(pRes));
 //  }
 //  taos_free_result(pRes);
 //
-//  pRes = taos_query(pConn, "use abc1");
+//  showDB(pConn);
+//
+//  pRes = taos_query(pConn, "create database abc1");
 //  if (taos_errno(pRes) != 0) {
-//    printf("error in use db, reason:%s\n", taos_errstr(pRes));
+//    printf("create to drop db, reason:%s\n", taos_errstr(pRes));
+//  }
+//  taos_free_result(pRes);
+//  taos_close(pConn);
+//}
+
+//TEST(testCase, create_stable_Test) {
+//  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+//  assert(pConn != NULL);
+//
+//  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2");
+//  if (taos_errno(pRes) != 0) {
+//    printf("error in create db, reason:%s\n", taos_errstr(pRes));
 //  }
 //  taos_free_result(pRes);
 //
-//  pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)");
+//  pRes = taos_query(pConn, "create table if not exists abc1.st1(ts timestamp, k int) tags(a int)");
 //  if (taos_errno(pRes) != 0) {
 //    printf("error in create stable, reason:%s\n", taos_errstr(pRes));
 //  }
@@ -279,40 +279,42 @@ TEST(testCase, connect_Test) {
 //  taos_free_result(pRes);
 //  taos_close(pConn);
 //}
-
-//TEST(testCase, create_table_Test) {
-//  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
-//  assert(pConn != NULL);
 //
-//  TAOS_RES* pRes = taos_query(pConn, "use abc1");
-//  taos_free_result(pRes);
+//  pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)");
+//  if (taos_errno(pRes) != 0) {
+//    printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes));
+//  }
 //
-//  pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)");
+//  TAOS_RES* pRes = taos_query(pConn, "use abc1");
 //  taos_free_result(pRes);
+//  pRes = taos_query(pConn, "drop stable `123_$^)`");
+//  if (taos_errno(pRes) != 0) {
+//    printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes));
+//  }
 //
 //  taos_close(pConn);
 //}
 
-TEST(testCase, create_ctable_Test) {
+TEST(testCase, create_table_Test) {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   assert(pConn != NULL);
 
   TAOS_RES* pRes = taos_query(pConn, "use abc1");
-  if (taos_errno(pRes) != 0) {
-    printf("failed to use db, reason:%s\n", taos_errstr(pRes));
-  }
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "create table tm0 using st1 tags(1)");
-  if (taos_errno(pRes) != 0) {
-    printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes));
-  }
+  pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k int)");
+  ASSERT_EQ(taos_errno(pRes), 0);
+
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k blob)");
+  ASSERT_NE(taos_errno(pRes), 0);
 
   taos_free_result(pRes);
   taos_close(pConn);
 }
 
-//TEST(testCase, show_stable_Test) {
+//TEST(testCase, create_ctable_Test) {
 //  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
 //  assert(pConn != NULL);
 //
@@ -322,26 +324,45 @@ TEST(testCase, create_ctable_Test) {
 //  }
 //  taos_free_result(pRes);
 //
-//  pRes = taos_query(pConn, "show stables");
+//  pRes = taos_query(pConn, "create table tm0 using st1 tags(1)");
 //  if (taos_errno(pRes) != 0) {
-//    printf("failed to show stables, reason:%s\n", taos_errstr(pRes));
-//    taos_free_result(pRes);
-//    ASSERT_TRUE(false);
-//  }
-//
-//  TAOS_ROW    pRow = NULL;
-//  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
-//  int32_t     numOfFields = taos_num_fields(pRes);
-//
-//  char str[512] = {0};
-//  while ((pRow = taos_fetch_row(pRes)) != NULL) {
-//    int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
-//    printf("%s\n", str);
+//    printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes));
 //  }
 //
 //  taos_free_result(pRes);
 //  taos_close(pConn);
 //}
+
+TEST(testCase, show_stable_Test) {
+  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  assert(pConn != nullptr);
+
+//  TAOS_RES* pRes = taos_query(pConn, "use abc1");
+//  if (taos_errno(pRes) != 0) {
+//    printf("failed to use db, reason:%s\n", taos_errstr(pRes));
+//  }
+//  taos_free_result(pRes);
+
+  TAOS_RES* pRes = taos_query(pConn, "show abc1.stables");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to show stables, reason:%s\n", taos_errstr(pRes));
+    taos_free_result(pRes);
+    ASSERT_TRUE(false);
+  }
+
+  TAOS_ROW    pRow = NULL;
+  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
+  int32_t     numOfFields = taos_num_fields(pRes);
+
+  char str[512] = {0};
+  while ((pRow = taos_fetch_row(pRes)) != NULL) {
+    int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
+    printf("%s\n", str);
+  }
+
+  taos_free_result(pRes);
+  taos_close(pConn);
+}
 //
 //TEST(testCase, show_vgroup_Test) {
 //  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
@@ -499,84 +520,108 @@ TEST(testCase, create_ctable_Test) {
 //
 //  taosHashCleanup(phash);
 //}
+//
+TEST(testCase, create_topic_Test) {
+  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  assert(pConn != NULL);
+
+  TAOS_RES* pRes = taos_query(pConn, "use abc1");
+  if (taos_errno(pRes) != 0) {
+    printf("error in use db, reason:%s\n", taos_errstr(pRes));
+  }
+  taos_free_result(pRes);
 
-// TEST(testCase, create_topic_Test) {
+  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
+  ASSERT_TRUE(pFields == nullptr);
+
+  int32_t numOfFields = taos_num_fields(pRes);
+  ASSERT_EQ(numOfFields, 0);
+
+  taos_free_result(pRes);
+
+  char* sql = "select * from tu";
+  pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql));
+  taos_free_result(pRes);
+  taos_close(pConn);
+}
+
+//TEST(testCase, insert_test) {
 //  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
-//  assert(pConn != NULL);
+//  ASSERT_NE(pConn, nullptr);
 //
-//  TAOS_RES* pRes = taos_query(pConn, "create database abc1");
-//  if (taos_errno(pRes) != 0) {
-//    printf("error in create db, reason:%s\n", taos_errstr(pRes));
-//  }
+//  TAOS_RES* pRes = taos_query(pConn, "use abc1");
 //  taos_free_result(pRes);
 //
-//  pRes = taos_query(pConn, "use abc1");
+//  pRes = taos_query(pConn, "insert into t_2 values(now, 1)");
 //  if (taos_errno(pRes) != 0) {
-//    printf("error in use db, reason:%s\n", taos_errstr(pRes));
-//  }
+//    printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes));
 //    taos_free_result(pRes);
-//
-//  pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)");
-//  if (taos_errno(pRes) != 0) {
-//    printf("error in create stable, reason:%s\n", taos_errstr(pRes));
+//    ASSERT_TRUE(false);
 //  }
 //
-//  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
-//  ASSERT_TRUE(pFields == NULL);
-//
-//  int32_t numOfFields = taos_num_fields(pRes);
-//  ASSERT_EQ(numOfFields, 0);
-//
 //  taos_free_result(pRes);
-//
-//  char* sql = "select * from st1";
-//  tmq_create_topic(pConn, "test_topic_1", sql, strlen(sql));
 //  taos_close(pConn);
 //}
 
-//TEST(testCase, insert_test) {
+//TEST(testCase, projection_query_tables) {
 //  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
-//  ASSERT_EQ(pConn, nullptr);
+//  ASSERT_NE(pConn, nullptr);
 //
 //  TAOS_RES* pRes = taos_query(pConn, "use abc1");
 //  taos_free_result(pRes);
 //
-//  pRes = taos_query(pConn, "insert into t_2 values(now, 1)");
+////  pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)");
+////  if (taos_errno(pRes) != 0) {
+////    printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
+////  }
+////  taos_free_result(pRes);
+////
+////  pRes = taos_query(pConn, "create table tu using st1 tags(1)");
+////  if (taos_errno(pRes) != 0) {
+////    printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
+////  }
+////  taos_free_result(pRes);
+////
+////  for(int32_t i = 0; i < 100; ++i) {
+////    char sql[512] = {0};
+////    sprintf(sql, "insert into tu values(now+%da, %d)", i, i);
+////    TAOS_RES* p = taos_query(pConn, sql);
+////    if (taos_errno(p) != 0) {
+////      printf("failed to insert data, reason:%s\n", taos_errstr(p));
+////    }
+////
+////    taos_free_result(p);
+////  }
+//
+//  pRes = taos_query(pConn, "select * from tu");
 //  if (taos_errno(pRes) != 0) {
-//    printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes));
+//    printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
 //    taos_free_result(pRes);
 //    ASSERT_TRUE(false);
 //  }
 //
+//  TAOS_ROW    pRow = NULL;
+//  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
+//  int32_t     numOfFields = taos_num_fields(pRes);
+//
+//  char str[512] = {0};
+//  while ((pRow = taos_fetch_row(pRes)) != NULL) {
+//    int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
+//    printf("%s\n", str);
+//  }
+//
 //  taos_free_result(pRes);
 //  taos_close(pConn);
 //}
 
-//TEST(testCase, projection_query_tables) {
+//TEST(testCase, projection_query_stables) {
 //  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
 //  ASSERT_NE(pConn, nullptr);
 //
-////  TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2");
-////  if (taos_errno(pRes) != 0) {
-////    printf("failed to use db, reason:%s\n", taos_errstr(pRes));
-////    taos_free_result(pRes);
-////    return;
-////  }
-//
-////  taos_free_result(pRes);
-//
 //  TAOS_RES* pRes = taos_query(pConn, "use abc1");
-//
-////  pRes = taos_query(pConn, "create table m1 (ts timestamp,  k int) tags(a int)");
 //  taos_free_result(pRes);
-////
-////  pRes = taos_query(pConn, "create table tu using m1 tags(1)");
-////  taos_free_result(pRes);
-////
-////  pRes = taos_query(pConn, "insert into tu values(now, 1)");
-////  taos_free_result(pRes);
 //
-//  pRes = taos_query(pConn, "select * from tu");
+//  pRes = taos_query(pConn, "select ts,k from m1");
 //  if (taos_errno(pRes) != 0) {
 //    printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
 //    taos_free_result(pRes);

source/common/src/tmsg.c

@@ -89,10 +89,10 @@ int tSerializeSClientHbReq(void **buf, const SClientHbReq *pReq) {
   int tlen = 0;
   tlen += taosEncodeSClientHbKey(buf, &pReq->connKey);
 
-  int kvNum = taosHashGetSize(pReq->info);
+  int32_t kvNum = taosHashGetSize(pReq->info);
   tlen += taosEncodeFixedI32(buf, kvNum);
   SKv kv;
-  void* pIter = taosHashIterate(pReq->info, pIter);
+  void* pIter = taosHashIterate(pReq->info, NULL);
   while (pIter != NULL) {
     taosHashGetKey(pIter, &kv.key, (size_t *)&kv.keyLen);
     kv.valueLen = taosHashGetDataLen(pIter);
@@ -104,14 +104,15 @@ int tSerializeSClientHbReq(void **buf, const SClientHbReq *pReq) {
   return tlen;
 }
 
-void *tDeserializeClientHbReq(void *buf, SClientHbReq *pReq) {
-  ASSERT(pReq->info != NULL);
+void *tDeserializeSClientHbReq(void *buf, SClientHbReq *pReq) {
   buf = taosDecodeSClientHbKey(buf, &pReq->connKey);
 
   // TODO: error handling
-  int kvNum;
-  taosDecodeFixedI32(buf, &kvNum);
+  int32_t kvNum;
+  buf = taosDecodeFixedI32(buf, &kvNum);
+  if (pReq->info == NULL) {
     pReq->info = taosHashInit(kvNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+  }
   for(int i = 0; i < kvNum; i++) {
     SKv kv;
     buf = taosDecodeSKv(buf, &kv);
@@ -121,12 +122,69 @@ void *tDeserializeClientHbReq(void *buf, SClientHbReq *pReq) {
   return buf;
 }
 
-int   tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pReq) {
+int   tSerializeSClientHbRsp(void** buf, const SClientHbRsp* pRsp) {
+  int tlen = 0;
+  tlen += taosEncodeSClientHbKey(buf, &pRsp->connKey);
+  tlen += taosEncodeFixedI32(buf, pRsp->status);
+  tlen += taosEncodeFixedI32(buf, pRsp->bodyLen);
+  tlen += taosEncodeBinary(buf, pRsp->body, pRsp->bodyLen);
+  return tlen;
+}
+void* tDeserializeSClientHbRsp(void* buf, SClientHbRsp* pRsp) {
+  buf = taosDecodeSClientHbKey(buf, &pRsp->connKey);
+  buf = taosDecodeFixedI32(buf, &pRsp->status);
+  buf = taosDecodeFixedI32(buf, &pRsp->bodyLen);
+  buf = taosDecodeBinary(buf, &pRsp->body, pRsp->bodyLen);
+  return buf;
+}
+
+int   tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pBatchReq) {
   int tlen = 0;
+  tlen += taosEncodeFixedI64(buf, pBatchReq->reqId);
+  int32_t reqNum = taosArrayGetSize(pBatchReq->reqs);
+  tlen += taosEncodeFixedI32(buf, reqNum); 
+  for (int i = 0; i < reqNum; i++) {
+    SClientHbReq* pReq = taosArrayGet(pBatchReq->reqs, i);
+    tlen += tSerializeSClientHbReq(buf, pReq);
+  }
   return tlen;
 }
 
-void* tDeserializeClientHbBatchReq(void* buf, SClientHbBatchReq* pReq) {
+void* tDeserializeSClientHbBatchReq(void* buf, SClientHbBatchReq* pBatchReq) {
+  buf = taosDecodeFixedI64(buf, &pBatchReq->reqId);
+  if (pBatchReq->reqs == NULL) {
+    pBatchReq->reqs = taosArrayInit(0, sizeof(SClientHbReq));
+  }
+  int32_t reqNum;
+  buf = taosDecodeFixedI32(buf, &reqNum);
+  for (int i = 0; i < reqNum; i++) {
+    SClientHbReq req = {0};
+    buf = tDeserializeSClientHbReq(buf, &req);
+    taosArrayPush(pBatchReq->reqs, &req);
+  }
+  return buf;
+}
+
+int   tSerializeSClientHbBatchRsp(void** buf, const SClientHbBatchRsp* pBatchRsp) {
+  int tlen = 0;
+  int32_t sz = taosArrayGetSize(pBatchRsp->rsps);
+  tlen += taosEncodeFixedI32(buf, sz);
+  for (int i = 0; i < sz; i++) {
+    SClientHbRsp* pRsp = taosArrayGet(pBatchRsp->rsps, i);
+    tlen += tSerializeSClientHbRsp(buf, pRsp);
+  }
+  return tlen;
+}
+
+void* tDeserializeSClientHbBatchRsp(void* buf, SClientHbBatchRsp* pBatchRsp) {
+  int32_t sz;
+  buf = taosDecodeFixedI32(buf, &sz);
+  pBatchRsp->rsps = taosArrayInit(sz, sizeof(SClientHbRsp));
+  for (int i = 0; i < sz; i++) {
+    SClientHbRsp rsp = {0};
+    buf = tDeserializeSClientHbRsp(buf, &rsp); 
+    taosArrayPush(pBatchRsp->rsps, &rsp);
+  }
   return buf;
 }
 

source/dnode/mgmt/impl/src/dndEnv.c

@@ -293,7 +293,7 @@ int32_t dndInit(const SDnodeEnvCfg *pCfg) {
   if (vnodeInit(&vnodeOpt) != 0) {
     dError("failed to init vnode since %s", terrstr());
     dndCleanup();
-    return NULL;
+    return -1;
   }
 
   memcpy(&dndEnv.cfg, pCfg, sizeof(SDnodeEnvCfg));

source/dnode/mgmt/impl/src/dndVnodes.c

@@ -421,6 +421,10 @@ static int32_t dndOpenVnodes(SDnode *pDnode) {
   pMgmt->totalVnodes = numOfVnodes;
 
   int32_t threadNum = pDnode->env.numOfCores;
+#if 1
+  threadNum = 1;
+#endif
+
   int32_t vnodesPerThread = numOfVnodes / threadNum + 1;
 
   SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread));

source/dnode/mgmt/impl/test/CMakeLists.txt

@@ -5,4 +5,5 @@ add_subdirectory(bnode)
 add_subdirectory(snode)
 add_subdirectory(mnode)
 add_subdirectory(vnode)
+add_subdirectory(stb)
 add_subdirectory(sut)

source/dnode/mgmt/impl/test/stb/CMakeLists.txt

+aux_source_directory(. DSTB_SRC)
+add_executable(dnode_test_stb ${DSTB_SRC})
+target_link_libraries(
+    dnode_test_stb
+    PUBLIC sut
+)
+
+add_test(
+    NAME dnode_test_stb
+    COMMAND dnode_test_stb
+)

source/dnode/mgmt/impl/test/stb/dstb.cpp

+/**
+ * @file db.cpp
+ * @author slguan (slguan@taosdata.com)
+ * @brief DNODE module vnode tests
+ * @version 0.1
+ * @date 2021-12-20
+ *
+ * @copyright Copyright (c) 2021
+ *
+ */
+
+#include "sut.h"
+
+class DndTestVnode : public ::testing::Test {
+ protected:
+  static void SetUpTestSuite() { test.Init("/tmp/dnode_test_stb", 9116); }
+  static void TearDownTestSuite() { test.Cleanup(); }
+
+  static Testbase test;
+
+ public:
+  void SetUp() override {}
+  void TearDown() override {}
+};
+
+Testbase DndTestVnode::test;
+
+TEST_F(DndTestVnode, 01_Create_Restart_Drop_Vnode) {
+  {
+    for (int i = 0; i < 3; ++i) {
+      int32_t contLen = sizeof(SCreateVnodeReq);
+
+      SCreateVnodeReq* pReq = (SCreateVnodeReq*)rpcMallocCont(contLen);
+      pReq->vgId = htonl(2);
+      pReq->dnodeId = htonl(1);
+      strcpy(pReq->db, "1.d1");
+      pReq->dbUid = htobe64(9527);
+      pReq->vgVersion = htonl(1);
+      pReq->cacheBlockSize = htonl(16);
+      pReq->totalBlocks = htonl(10);
+      pReq->daysPerFile = htonl(10);
+      pReq->daysToKeep0 = htonl(3650);
+      pReq->daysToKeep1 = htonl(3650);
+      pReq->daysToKeep2 = htonl(3650);
+      pReq->minRows = htonl(100);
+      pReq->minRows = htonl(4096);
+      pReq->commitTime = htonl(3600);
+      pReq->fsyncPeriod = htonl(3000);
+      pReq->walLevel = 1;
+      pReq->precision = 0;
+      pReq->compression = 2;
+      pReq->replica = 1;
+      pReq->quorum = 1;
+      pReq->update = 0;
+      pReq->cacheLastRow = 0;
+      pReq->selfIndex = 0;
+      for (int r = 0; r < pReq->replica; ++r) {
+        SReplica* pReplica = &pReq->replicas[r];
+        pReplica->id = htonl(1);
+        pReplica->port = htons(9527);
+      }
+
+      SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_VNODE, pReq, contLen);
+      ASSERT_NE(pRsp, nullptr);
+      if (i == 0) {
+        ASSERT_EQ(pRsp->code, 0);
+        test.Restart();
+      } else {
+        ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED);
+      }
+    }
+  }
+
+  {
+    for (int i = 0; i < 3; ++i) {
+      int32_t contLen = sizeof(SAlterVnodeReq);
+
+      SAlterVnodeReq* pReq = (SAlterVnodeReq*)rpcMallocCont(contLen);
+      pReq->vgId = htonl(2);
+      pReq->dnodeId = htonl(1);
+      strcpy(pReq->db, "1.d1");
+      pReq->dbUid = htobe64(9527);
+      pReq->vgVersion = htonl(2);
+      pReq->cacheBlockSize = htonl(16);
+      pReq->totalBlocks = htonl(10);
+      pReq->daysPerFile = htonl(10);
+      pReq->daysToKeep0 = htonl(3650);
+      pReq->daysToKeep1 = htonl(3650);
+      pReq->daysToKeep2 = htonl(3650);
+      pReq->minRows = htonl(100);
+      pReq->minRows = htonl(4096);
+      pReq->commitTime = htonl(3600);
+      pReq->fsyncPeriod = htonl(3000);
+      pReq->walLevel = 1;
+      pReq->precision = 0;
+      pReq->compression = 2;
+      pReq->replica = 1;
+      pReq->quorum = 1;
+      pReq->update = 0;
+      pReq->cacheLastRow = 0;
+      pReq->selfIndex = 0;
+      for (int r = 0; r < pReq->replica; ++r) {
+        SReplica* pReplica = &pReq->replicas[r];
+        pReplica->id = htonl(1);
+        pReplica->port = htons(9527);
+      }
+
+      SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_VNODE, pReq, contLen);
+      ASSERT_NE(pRsp, nullptr);
+      ASSERT_EQ(pRsp->code, 0);
+    }
+  }
+
+  {
+    for (int i = 0; i < 3; ++i) {
+      int32_t contLen = sizeof(SDropVnodeReq);
+
+      SDropVnodeReq* pReq = (SDropVnodeReq*)rpcMallocCont(contLen);
+      pReq->vgId = htonl(2);
+      pReq->dnodeId = htonl(1);
+      strcpy(pReq->db, "1.d1");
+      pReq->dbUid = htobe64(9527);
+
+      SRpcMsg rpcMsg = {0};
+      rpcMsg.pCont = pReq;
+      rpcMsg.contLen = sizeof(SDropVnodeReq);
+      rpcMsg.msgType = TDMT_DND_DROP_VNODE;
+
+      SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_VNODE, pReq, contLen);
+      ASSERT_NE(pRsp, nullptr);
+      if (i == 0) {
+        ASSERT_EQ(pRsp->code, 0);
+        test.Restart();
+      } else {
+        ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_NOT_DEPLOYED);
+      }
+    }
+  }
+}

source/dnode/mnode/impl/inc/mndDef.h

@@ -346,19 +346,23 @@ typedef struct SMqTopicObj {
   char     *logicalPlan;
   char     *physicalPlan;
   SHashObj *cgroups;  // SHashObj<SMqCGroup>
+  SHashObj *consumers;  // SHashObj<SMqConsumerObj>
 } SMqTopicObj;
 
 // TODO: add cache and change name to id
 typedef struct SMqConsumerTopic {
-  char   name[TSDB_TOPIC_FNAME_LEN];
+  int32_t epoch;
+  char    name[TSDB_TOPIC_NAME_LEN];
+  //TODO: replace with something with ep
   SList   *vgroups;  // SList<int32_t>
 } SMqConsumerTopic;
 
 typedef struct SMqConsumerObj {
-  SRWLatch lock;
   int64_t  consumerId;
+  SRWLatch lock;
   char     cgroup[TSDB_CONSUMER_GROUP_LEN];
   SArray  *topics;  // SArray<SMqConsumerTopic>
+  SHashObj *topicHash;
 } SMqConsumerObj;
 
 typedef struct SMqSubConsumerObj {
@@ -409,7 +413,7 @@ typedef struct SMqVGroupHbObj {
 
 #if 0
 typedef struct SCGroupObj {
-  char     name[TSDB_TOPIC_FNAME_LEN];
+  char     name[TSDB_TOPIC_NAME_LEN];
   int64_t  createTime;
   int64_t  updateTime;
   uint64_t uid;

source/dnode/mnode/impl/src/mndDb.c

@@ -613,32 +613,14 @@ static int32_t mndSetUpdateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj
 static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pReq, SDbObj *pOld, SDbObj *pNew) {
   int32_t code = -1;
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pReq->rpcMsg);
-  if (pTrans == NULL) {
-    mError("db:%s, failed to update since %s", pOld->name, terrstr());
-    return terrno;
-  }
+  if (pTrans == NULL) goto UPDATE_DB_OVER;
 
   mDebug("trans:%d, used to update db:%s", pTrans->id, pOld->name);
 
-  if (mndSetUpdateDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) {
-    mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr());
-    goto UPDATE_DB_OVER;
-  }
-
-  if (mndSetUpdateDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) {
-    mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr());
-    goto UPDATE_DB_OVER;
-  }
-
-  if (mndSetUpdateDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) {
-    mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
-    goto UPDATE_DB_OVER;
-  }
-
-  if (mndTransPrepare(pMnode, pTrans) != 0) {
-    mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
-    goto UPDATE_DB_OVER;
-  }
+  if (mndSetUpdateDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER;
+  if (mndSetUpdateDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER;
+  if (mndSetUpdateDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER;
+  if (mndTransPrepare(pMnode, pTrans) != 0) goto UPDATE_DB_OVER;
 
   code = 0;
 

source/dnode/mnode/impl/src/mndProfile.c

@@ -15,10 +15,13 @@
 
 #define _DEFAULT_SOURCE
 #include "mndProfile.h"
+#include "mndConsumer.h"
 #include "mndDb.h"
 #include "mndMnode.h"
 #include "mndShow.h"
+#include "mndTopic.h"
 #include "mndUser.h"
+#include "mndVgroup.h"
 
 #define QUERY_ID_SIZE 20
 #define QUERY_OBJ_ID_SIZE 18
@@ -180,6 +183,11 @@ static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) {
 
 static int32_t mndProcessConnectReq(SMnodeMsg *pReq) {
   SMnode   *pMnode = pReq->pMnode;
+  SUserObj *pUser = NULL;
+  SDbObj   *pDb = NULL;
+  SConnObj *pConn = NULL;
+  int32_t   code = -1;
+
   SConnectReq *pConnReq = pReq->rpcMsg.pCont;
   pConnReq->pid = htonl(pConnReq->pid);
   pConnReq->startTime = htobe64(pConnReq->startTime);
@@ -187,54 +195,61 @@ static int32_t mndProcessConnectReq(SMnodeMsg *pReq) {
   SRpcConnInfo info = {0};
   if (rpcGetConnInfo(pReq->rpcMsg.handle, &info) != 0) {
     mError("user:%s, failed to login while get connection info since %s", pReq->user, terrstr());
-    return -1;
+    goto CONN_OVER;
   }
 
   char ip[30];
   taosIp2String(info.clientIp, ip);
 
+  pUser = mndAcquireUser(pMnode, pReq->user);
+  if (pUser == NULL) {
+    mError("user:%s, failed to login while acquire user since %s", pReq->user, terrstr());
+    goto CONN_OVER;
+  }
+
   if (pConnReq->db[0]) {
-    snprintf(pReq->db, TSDB_DB_FNAME_LEN, "%d%s%s", pReq->acctId, TS_PATH_DELIMITER, pConnReq->db);
-    SDbObj *pDb = mndAcquireDb(pMnode, pReq->db);
+    snprintf(pReq->db, TSDB_DB_FNAME_LEN, "%d%s%s", pUser->acctId, TS_PATH_DELIMITER, pConnReq->db);
+    pDb = mndAcquireDb(pMnode, pReq->db);
     if (pDb == NULL) {
       terrno = TSDB_CODE_MND_INVALID_DB;
       mError("user:%s, failed to login from %s while use db:%s since %s", pReq->user, ip, pConnReq->db, terrstr());
-      return -1;
+      goto CONN_OVER;
     }
-    mndReleaseDb(pMnode, pDb);
   }
 
-  SConnObj *pConn = mndCreateConn(pMnode, &info, pConnReq->pid, pConnReq->app, pConnReq->startTime);
+  pConn = mndCreateConn(pMnode, &info, pConnReq->pid, pConnReq->app, pConnReq->startTime);
   if (pConn == NULL) {
     mError("user:%s, failed to login from %s while create connection since %s", pReq->user, ip, terrstr());
-    return -1;
+    goto CONN_OVER;
   }
 
   SConnectRsp *pRsp = rpcMallocCont(sizeof(SConnectRsp));
   if (pRsp == NULL) {
-    mndReleaseConn(pMnode, pConn);
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     mError("user:%s, failed to login from %s while create rsp since %s", pReq->user, ip, terrstr());
-    return -1;
+    goto CONN_OVER;
   }
 
-  SUserObj *pUser = mndAcquireUser(pMnode, pReq->user);
-  if (pUser != NULL) {
   pRsp->acctId = htonl(pUser->acctId);
   pRsp->superUser = pUser->superUser;
-    mndReleaseUser(pMnode, pUser);
-  }
-
   pRsp->clusterId = htobe64(pMnode->clusterId);
   pRsp->connId = htonl(pConn->id);
   mndGetMnodeEpSet(pMnode, &pRsp->epSet);
-  mndReleaseConn(pMnode, pConn);
 
   pReq->contLen = sizeof(SConnectRsp);
   pReq->pCont = pRsp;
 
   mDebug("user:%s, login from %s, conn:%d, app:%s", info.user, ip, pConn->id, pConnReq->app);
-  return 0;
+
+  code = 0;
+
+CONN_OVER:
+
+  mndReleaseUser(pMnode, pUser);
+  mndReleaseDb(pMnode, pDb);
+  mndReleaseConn(pMnode, pConn);
+
+  return code;
 }
 
 static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatReq *pReq) {
@@ -257,7 +272,100 @@ static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatReq *pReq) {
   return TSDB_CODE_SUCCESS;
 }
 
+static SClientHbRsp* mndMqHbBuildRsp(SMnode* pMnode, SClientHbReq* pReq) {
+  SClientHbRsp* pRsp = malloc(sizeof(SClientHbRsp));
+  if (pRsp == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+  pRsp->connKey = pReq->connKey;
+  SMqHbBatchRsp batchRsp;
+  batchRsp.batchRsps = taosArrayInit(0, sizeof(SMqHbRsp));
+  if (batchRsp.batchRsps == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+  SClientHbKey connKey = pReq->connKey;
+  SHashObj* pObj =  pReq->info;
+  SKv* pKv = taosHashGet(pObj, "mq-tmp", strlen("mq-tmp") + 1);
+  if (pKv == NULL) {
+    free(pRsp);
+    return NULL;
+  }
+  SMqHbMsg mqHb;
+  taosDecodeSMqMsg(pKv->value, &mqHb);
+  /*int64_t clientUid = htonl(pKv->value);*/
+  /*if (mqHb.epoch )*/
+  int sz = taosArrayGetSize(mqHb.pTopics);
+  SMqConsumerObj* pConsumer = mndAcquireConsumer(pMnode, mqHb.consumerId); 
+  for (int i = 0; i < sz; i++) {
+    SMqHbOneTopicBatchRsp innerBatchRsp;
+    innerBatchRsp.rsps = taosArrayInit(sz, sizeof(SMqHbRsp));
+    if (innerBatchRsp.rsps == NULL) {
+      //TODO
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return NULL;
+    }
+    SMqHbTopicInfo* topicInfo = taosArrayGet(mqHb.pTopics, i);
+    SMqConsumerTopic* pConsumerTopic = taosHashGet(pConsumer->topicHash, topicInfo->name, strlen(topicInfo->name)+1);
+    if (pConsumerTopic->epoch != topicInfo->epoch) {
+      //add new vgids into rsp
+      int vgSz = taosArrayGetSize(topicInfo->pVgInfo);
+      for (int j = 0; j < vgSz; j++) {
+        SMqHbRsp innerRsp;
+        SMqHbVgInfo* pVgInfo = taosArrayGet(topicInfo->pVgInfo, i);
+        SVgObj* pVgObj = mndAcquireVgroup(pMnode, pVgInfo->vgId);
+        innerRsp.epSet = mndGetVgroupEpset(pMnode, pVgObj);
+        taosArrayPush(innerBatchRsp.rsps, &innerRsp);
+      }
+    }
+    taosArrayPush(batchRsp.batchRsps, &innerBatchRsp);
+  }
+  int32_t tlen = taosEncodeSMqHbBatchRsp(NULL, &batchRsp);
+  void* buf = malloc(tlen);
+  if (buf == NULL) {
+    //TODO
+    return NULL;
+  }
+  void* abuf = buf;
+  taosEncodeSMqHbBatchRsp(&abuf, &batchRsp);
+  pRsp->body = buf;
+  pRsp->bodyLen = tlen;
+  return pRsp;
+}
+
 static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) {
+  SMnode *pMnode = pReq->pMnode;
+  char *batchReqStr = pReq->rpcMsg.pCont;
+  SClientHbBatchReq batchReq = {0};
+  tDeserializeSClientHbBatchReq(batchReqStr, &batchReq);
+  SArray *pArray = batchReq.reqs;
+  int sz = taosArrayGetSize(pArray);
+
+  SClientHbBatchRsp batchRsp = {0};
+  batchRsp.rsps = taosArrayInit(0, sizeof(SClientHbRsp));
+
+  for (int i = 0; i < sz; i++) {
+    SClientHbReq* pHbReq = taosArrayGet(pArray, i);
+    if (pHbReq->connKey.hbType == HEARTBEAT_TYPE_QUERY) {
+
+    } else if (pHbReq->connKey.hbType == HEARTBEAT_TYPE_MQ) {
+      SClientHbRsp *pRsp = mndMqHbBuildRsp(pMnode, pHbReq);
+      if (pRsp != NULL) {
+        taosArrayPush(batchRsp.rsps, pRsp);
+        free(pRsp);
+      }
+    }
+  }
+  int32_t tlen = tSerializeSClientHbBatchRsp(NULL, &batchRsp);
+  void* buf = rpcMallocCont(tlen);
+  void* abuf = buf;
+  tSerializeSClientHbBatchRsp(&abuf, &batchRsp);
+  pReq->contLen = tlen;
+  pReq->pCont = buf;
+  return 0;
+
+#if 0
   SMnode       *pMnode = pReq->pMnode;
   SProfileMgmt *pMgmt = &pMnode->profileMgmt;
 
@@ -327,6 +435,7 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) {
   pReq->contLen = sizeof(SConnectRsp);
   pReq->pCont = pRsp;
   return 0;
+#endif
 }
 
 static int32_t mndProcessKillQueryReq(SMnodeMsg *pReq) {

source/dnode/mnode/impl/src/mndStb.c

@@ -31,16 +31,16 @@ static SSdbRaw *mndStbActionEncode(SStbObj *pStb);
 static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw);
 static int32_t  mndStbActionInsert(SSdb *pSdb, SStbObj *pStb);
 static int32_t  mndStbActionDelete(SSdb *pSdb, SStbObj *pStb);
-static int32_t  mndStbActionUpdate(SSdb *pSdb, SStbObj *pOldStb, SStbObj *pNewStb);
-static int32_t  mndProcesSMCreateStbReq(SMnodeMsg *pMsg);
-static int32_t  mndProcesSMAlterStbReq(SMnodeMsg *pMsg);
-static int32_t  mndProcesSMDropStbReq(SMnodeMsg *pMsg);
-static int32_t  mndProcessCreateStbInRsp(SMnodeMsg *pMsg);
-static int32_t  mndProcessAlterStbInRsp(SMnodeMsg *pMsg);
-static int32_t  mndProcessDropStbInRsp(SMnodeMsg *pMsg);
-static int32_t  mndProcessStbMetaMsg(SMnodeMsg *pMsg);
-static int32_t  mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta);
-static int32_t  mndRetrieveStb(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows);
+static int32_t  mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew);
+static int32_t  mndProcessMCreateStbReq(SMnodeMsg *pReq);
+static int32_t  mndProcessMAlterStbReq(SMnodeMsg *pReq);
+static int32_t  mndProcessMDropStbReq(SMnodeMsg *pReq);
+static int32_t  mndProcessVCreateStbRsp(SMnodeMsg *pRsp);
+static int32_t  mndProcessVAlterStbRsp(SMnodeMsg *pRsp);
+static int32_t  mndProcessVDropStbRsp(SMnodeMsg *pRsp);
+static int32_t  mndProcessStbMetaReq(SMnodeMsg *pReq);
+static int32_t  mndGetStbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta);
+static int32_t  mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows);
 static void     mndCancelGetNextStb(SMnode *pMnode, void *pIter);
 
 int32_t mndInitStb(SMnode *pMnode) {
@@ -52,13 +52,13 @@ int32_t mndInitStb(SMnode *pMnode) {
                      .updateFp = (SdbUpdateFp)mndStbActionUpdate,
                      .deleteFp = (SdbDeleteFp)mndStbActionDelete};
 
-  mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STB, mndProcesSMCreateStbReq);
-  mndSetMsgHandle(pMnode, TDMT_MND_ALTER_STB, mndProcesSMAlterStbReq);
-  mndSetMsgHandle(pMnode, TDMT_MND_DROP_STB, mndProcesSMDropStbReq);
-  mndSetMsgHandle(pMnode, TDMT_VND_CREATE_STB_RSP, mndProcessCreateStbInRsp);
-  mndSetMsgHandle(pMnode, TDMT_VND_ALTER_STB_RSP, mndProcessAlterStbInRsp);
-  mndSetMsgHandle(pMnode, TDMT_VND_DROP_STB_RSP, mndProcessDropStbInRsp);
-  mndSetMsgHandle(pMnode, TDMT_MND_STB_META, mndProcessStbMetaMsg);
+  mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STB, mndProcessMCreateStbReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_ALTER_STB, mndProcessMAlterStbReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_DROP_STB, mndProcessMDropStbReq);
+  mndSetMsgHandle(pMnode, TDMT_VND_CREATE_STB_RSP, mndProcessVCreateStbRsp);
+  mndSetMsgHandle(pMnode, TDMT_VND_ALTER_STB_RSP, mndProcessVAlterStbRsp);
+  mndSetMsgHandle(pMnode, TDMT_VND_DROP_STB_RSP, mndProcessVDropStbRsp);
+  mndSetMsgHandle(pMnode, TDMT_MND_STB_META, mndProcessStbMetaReq);
 
   mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_STB, mndGetStbMeta);
   mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STB, mndRetrieveStb);
@@ -177,27 +177,27 @@ static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb) {
   return 0;
 }
 
-static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOldStb, SStbObj *pNewStb) {
-  mTrace("stb:%s, perform update action, old row:%p new row:%p", pOldStb->name, pOldStb, pNewStb);
-  atomic_exchange_32(&pOldStb->updateTime, pNewStb->updateTime);
-  atomic_exchange_32(&pOldStb->version, pNewStb->version);
+static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) {
+  mTrace("stb:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew);
+  atomic_exchange_32(&pOld->updateTime, pNew->updateTime);
+  atomic_exchange_32(&pOld->version, pNew->version);
 
-  taosWLockLatch(&pOldStb->lock);
-  pOldStb->numOfColumns = pNewStb->numOfColumns;
-  pOldStb->numOfTags = pNewStb->numOfTags;
-  int32_t totalCols = pNewStb->numOfTags + pNewStb->numOfColumns;
+  taosWLockLatch(&pOld->lock);
+  pOld->numOfColumns = pNew->numOfColumns;
+  pOld->numOfTags = pNew->numOfTags;
+  int32_t totalCols = pNew->numOfTags + pNew->numOfColumns;
   int32_t totalSize = totalCols * sizeof(SSchema);
 
-  if (pOldStb->numOfTags + pOldStb->numOfColumns < totalCols) {
+  if (pOld->numOfTags + pOld->numOfColumns < totalCols) {
     void *pSchema = malloc(totalSize);
     if (pSchema != NULL) {
-      free(pOldStb->pSchema);
-      pOldStb->pSchema = pSchema;
+      free(pOld->pSchema);
+      pOld->pSchema = pSchema;
     }
   }
 
-  memcpy(pOldStb->pSchema, pNewStb->pSchema, totalSize);
-  taosWUnLockLatch(&pOldStb->lock);
+  memcpy(pOld->pSchema, pNew->pSchema, totalSize);
+  taosWUnLockLatch(&pOld->lock);
   return 0;
 }
 
@@ -215,7 +215,7 @@ void mndReleaseStb(SMnode *pMnode, SStbObj *pStb) {
   sdbRelease(pSdb, pStb);
 }
 
-static SDbObj *mndAcquireDbByStb(SMnode *pMnode, char *stbName) {
+static SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) {
   SName name = {0};
   tNameFromString(&name, stbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
 
@@ -225,17 +225,17 @@ static SDbObj *mndAcquireDbByStb(SMnode *pMnode, char *stbName) {
   return mndAcquireDb(pMnode, db);
 }
 
-static void *mndBuildCreateStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int *pContLen) {
+static void *mndBuildCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) {
   SVCreateTbReq req;
   void         *buf;
-  int           bsize;
+  int32_t       bsize;
   SMsgHead     *pMsgHead;
 
   req.ver = 0;
   SName name = {0};
-  tNameFromString(&name, pStb->name, T_NAME_ACCT|T_NAME_DB|T_NAME_TABLE);
+  tNameFromString(&name, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
 
-  req.name = (char*) tNameGetTableName(&name);
+  req.name = (char *)tNameGetTableName(&name);
   req.ttl = 0;
   req.keep = 0;
   req.type = TD_SUPER_TABLE;
@@ -264,7 +264,7 @@ static void *mndBuildCreateStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb
   return buf;
 }
 
-static SVDropTbReq *mndBuildDropStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb) {
+static SVDropTbReq *mndBuildDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb) {
   int32_t contLen = sizeof(SVDropTbReq);
 
   SVDropTbReq *pDrop = calloc(1, contLen);
@@ -276,12 +276,12 @@ static SVDropTbReq *mndBuildDropStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj
   pDrop->head.contLen = htonl(contLen);
   pDrop->head.vgId = htonl(pVgroup->vgId);
   memcpy(pDrop->name, pStb->name, TSDB_TABLE_FNAME_LEN);
-  // pDrop->suid = htobe64(pStb->uid);
+  pDrop->suid = htobe64(pStb->uid);
 
   return pDrop;
 }
 
-static int32_t mndCheckCreateStbMsg(SMCreateStbReq *pCreate) {
+static int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) {
   pCreate->numOfColumns = htonl(pCreate->numOfColumns);
   pCreate->numOfTags = htonl(pCreate->numOfTags);
   int32_t totalCols = pCreate->numOfColumns + pCreate->numOfTags;
@@ -356,15 +356,15 @@ static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj
   SSdb   *pSdb = pMnode->pSdb;
   SVgObj *pVgroup = NULL;
   void   *pIter = NULL;
-  int     contLen;
+  int32_t     contLen;
 
   while (1) {
     pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
     if (pIter == NULL) break;
     if (pVgroup->dbUid != pDb->uid) continue;
 
-    void *pMsg = mndBuildCreateStbMsg(pMnode, pVgroup, pStb, &contLen);
-    if (pMsg == NULL) {
+    void *pReq = mndBuildCreateStbReq(pMnode, pVgroup, pStb, &contLen);
+    if (pReq == NULL) {
       sdbCancelFetch(pSdb, pIter);
       sdbRelease(pSdb, pVgroup);
       terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -373,11 +373,11 @@ static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj
 
     STransAction action = {0};
     action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
-    action.pCont = pMsg;
+    action.pCont = pReq;
     action.contLen = contLen;
     action.msgType = TDMT_VND_CREATE_STB;
     if (mndTransAppendRedoAction(pTrans, &action) != 0) {
-      free(pMsg);
+      free(pReq);
       sdbCancelFetch(pSdb, pIter);
       sdbRelease(pSdb, pVgroup);
       return -1;
@@ -398,8 +398,8 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj
     if (pIter == NULL) break;
     if (pVgroup->dbUid != pDb->uid) continue;
 
-    SVDropTbReq *pMsg = mndBuildDropStbMsg(pMnode, pVgroup, pStb);
-    if (pMsg == NULL) {
+    SVDropTbReq *pReq = mndBuildDropStbReq(pMnode, pVgroup, pStb);
+    if (pReq == NULL) {
       sdbCancelFetch(pSdb, pIter);
       sdbRelease(pSdb, pVgroup);
       terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -408,11 +408,11 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj
 
     STransAction action = {0};
     action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
-    action.pCont = pMsg;
+    action.pCont = pReq;
     action.contLen = sizeof(SVDropTbReq);
     action.msgType = TDMT_VND_DROP_STB;
     if (mndTransAppendUndoAction(pTrans, &action) != 0) {
-      free(pMsg);
+      free(pReq);
       sdbCancelFetch(pSdb, pIter);
       sdbRelease(pSdb, pVgroup);
       return -1;
@@ -423,7 +423,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj
   return 0;
 }
 
-static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SMCreateStbReq *pCreate, SDbObj *pDb) {
+static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) {
   SStbObj stbObj = {0};
   tstrncpy(stbObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN);
   tstrncpy(stbObj.db, pDb->name, TSDB_DB_FNAME_LEN);
@@ -449,43 +449,17 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SMCreateStbReq *pCr
   }
 
   int32_t code = 0;
-  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg);
-  if (pTrans == NULL) {
-    mError("stb:%s, failed to create since %s", pCreate->name, terrstr());
-    return -1;
-  }
-  mDebug("trans:%d, used to create stb:%s", pTrans->id, pCreate->name);
-
-  if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) {
-    mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr());
-    goto CREATE_STB_OVER;
-  }
-
-  if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, &stbObj) != 0) {
-    mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr());
-    goto CREATE_STB_OVER;
-  }
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg);
+  if (pTrans == NULL) goto CREATE_STB_OVER;
 
-  if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) {
-    mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr());
-    goto CREATE_STB_OVER;
-  }
-
-  if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) {
-    mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
-    goto CREATE_STB_OVER;
-  }
+  mDebug("trans:%d, used to create stb:%s", pTrans->id, pCreate->name);
 
-  if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, &stbObj) != 0) {
-    mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
-    goto CREATE_STB_OVER;
-  }
-
-  if (mndTransPrepare(pMnode, pTrans) != 0) {
-    mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
-    mndTransDrop(pTrans);
-    return -1;
-  }
+  if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto CREATE_STB_OVER;
+  if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto CREATE_STB_OVER;
+  if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto CREATE_STB_OVER;
+  if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto CREATE_STB_OVER;
+  if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto CREATE_STB_OVER;
+  if (mndTransPrepare(pMnode, pTrans) != 0) goto CREATE_STB_OVER;
 
   code = 0;
 
@@ -494,13 +468,13 @@ CREATE_STB_OVER:
   return code;
 }
 
-static int32_t mndProcesSMCreateStbReq(SMnodeMsg *pMsg) {
-  SMnode        *pMnode = pMsg->pMnode;
-  SMCreateStbReq *pCreate = pMsg->rpcMsg.pCont;
+static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) {
+  SMnode         *pMnode = pReq->pMnode;
+  SMCreateStbReq *pCreate = pReq->rpcMsg.pCont;
 
   mDebug("stb:%s, start to create", pCreate->name);
 
-  if (mndCheckCreateStbMsg(pCreate) != 0) {
+  if (mndCheckCreateStbReq(pCreate) != 0) {
     mError("stb:%s, failed to create since %s", pCreate->name, terrstr());
     return -1;
   }
@@ -536,7 +510,7 @@ static int32_t mndProcesSMCreateStbReq(SMnodeMsg *pMsg) {
     return -1;
   }
 
-  int32_t code = mndCreateStb(pMnode, pMsg, pCreate, pDb);
+  int32_t code = mndCreateStb(pMnode, pReq, pCreate, pDb);
   mndReleaseDb(pMnode, pDb);
 
   if (code != 0) {
@@ -548,12 +522,12 @@ static int32_t mndProcesSMCreateStbReq(SMnodeMsg *pMsg) {
   return TSDB_CODE_MND_ACTION_IN_PROGRESS;
 }
 
-static int32_t mndProcessCreateStbInRsp(SMnodeMsg *pMsg) {
-  mndTransProcessRsp(pMsg);
+static int32_t mndProcessVCreateStbRsp(SMnodeMsg *pRsp) {
+  mndTransProcessRsp(pRsp);
   return 0;
 }
 
-static int32_t mndCheckAlterStbMsg(SMAlterStbReq *pAlter) {
+static int32_t mndCheckAlterStbReq(SMAlterStbReq *pAlter) {
   SSchema *pSchema = &pAlter->schema;
   pSchema->colId = htonl(pSchema->colId);
   pSchema->bytes = htonl(pSchema->bytes);
@@ -578,15 +552,15 @@ static int32_t mndCheckAlterStbMsg(SMAlterStbReq *pAlter) {
   return 0;
 }
 
-static int32_t mndUpdateStb(SMnode *pMnode, SMnodeMsg *pMsg, SStbObj *pOldStb, SStbObj *pNewStb) { return 0; }
+static int32_t mndUpdateStb(SMnode *pMnode, SMnodeMsg *pReq, SStbObj *pOld, SStbObj *pNew) { return 0; }
 
-static int32_t mndProcesSMAlterStbReq(SMnodeMsg *pMsg) {
-  SMnode       *pMnode = pMsg->pMnode;
-  SMAlterStbReq *pAlter = pMsg->rpcMsg.pCont;
+static int32_t mndProcessMAlterStbReq(SMnodeMsg *pReq) {
+  SMnode        *pMnode = pReq->pMnode;
+  SMAlterStbReq *pAlter = pReq->rpcMsg.pCont;
 
   mDebug("stb:%s, start to alter", pAlter->name);
 
-  if (mndCheckAlterStbMsg(pAlter) != 0) {
+  if (mndCheckAlterStbReq(pAlter) != 0) {
     mError("stb:%s, failed to alter since %s", pAlter->name, terrstr());
     return -1;
   }
@@ -601,7 +575,7 @@ static int32_t mndProcesSMAlterStbReq(SMnodeMsg *pMsg) {
   SStbObj stbObj = {0};
   memcpy(&stbObj, pStb, sizeof(SStbObj));
 
-  int32_t code = mndUpdateStb(pMnode, pMsg, pStb, &stbObj);
+  int32_t code = mndUpdateStb(pMnode, pReq, pStb, &stbObj);
   mndReleaseStb(pMnode, pStb);
 
   if (code != 0) {
@@ -612,8 +586,8 @@ static int32_t mndProcesSMAlterStbReq(SMnodeMsg *pMsg) {
   return TSDB_CODE_MND_ACTION_IN_PROGRESS;
 }
 
-static int32_t mndProcessAlterStbInRsp(SMnodeMsg *pMsg) {
-  mndTransProcessRsp(pMsg);
+static int32_t mndProcessVAlterStbRsp(SMnodeMsg *pRsp) {
+  mndTransProcessRsp(pRsp);
   return 0;
 }
 
@@ -648,44 +622,19 @@ static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SStbObj
 
 static int32_t mndSetDropStbUndoActions(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { return 0; }
 
-static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pMsg, SStbObj *pStb) {
+static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pReq, SStbObj *pStb) {
   int32_t code = -1;
-  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg);
-  if (pTrans == NULL) {
-    mError("stb:%s, failed to drop since %s", pStb->name, terrstr());
-    return -1;
-  }
-  mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name);
-
-  if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) {
-    mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr());
-    goto DROP_STB_OVER;
-  }
-
-  if (mndSetDropStbUndoLogs(pMnode, pTrans, pStb) != 0) {
-    mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr());
-    goto DROP_STB_OVER;
-  }
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg);
+  if (pTrans == NULL)goto DROP_STB_OVER;
 
-  if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) {
-    mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr());
-    goto DROP_STB_OVER;
-  }
-
-  if (mndSetDropStbRedoActions(pMnode, pTrans, pStb) != 0) {
-    mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
-    goto DROP_STB_OVER;
-  }
-
-  if (mndSetDropStbUndoActions(pMnode, pTrans, pStb) != 0) {
-    mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
-    goto DROP_STB_OVER;
-  }
+  mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name);
 
-  if (mndTransPrepare(pMnode, pTrans) != 0) {
-    mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
-    goto DROP_STB_OVER;
-  }
+  if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER;
+  if (mndSetDropStbUndoLogs(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER;
+  if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER;
+  if (mndSetDropStbRedoActions(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER;
+  if (mndSetDropStbUndoActions(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER;
+  if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_STB_OVER;
 
   code = 0;
 
@@ -694,9 +643,9 @@ DROP_STB_OVER:
   return 0;
 }
 
-static int32_t mndProcesSMDropStbReq(SMnodeMsg *pMsg) {
-  SMnode      *pMnode = pMsg->pMnode;
-  SMDropStbReq *pDrop = pMsg->rpcMsg.pCont;
+static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq) {
+  SMnode       *pMnode = pReq->pMnode;
+  SMDropStbReq *pDrop = pReq->rpcMsg.pCont;
 
   mDebug("stb:%s, start to drop", pDrop->name);
 
@@ -712,7 +661,7 @@ static int32_t mndProcesSMDropStbReq(SMnodeMsg *pMsg) {
     }
   }
 
-  int32_t code = mndDropStb(pMnode, pMsg, pStb);
+  int32_t code = mndDropStb(pMnode, pReq, pStb);
   mndReleaseStb(pMnode, pStb);
 
   if (code != 0) {
@@ -724,14 +673,14 @@ static int32_t mndProcesSMDropStbReq(SMnodeMsg *pMsg) {
   return TSDB_CODE_MND_ACTION_IN_PROGRESS;
 }
 
-static int32_t mndProcessDropStbInRsp(SMnodeMsg *pMsg) {
-  mndTransProcessRsp(pMsg);
+static int32_t mndProcessVDropStbRsp(SMnodeMsg *pRsp) {
+  mndTransProcessRsp(pRsp);
   return 0;
 }
 
-static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg) {
-  SMnode        *pMnode = pMsg->pMnode;
-  STableInfoReq *pInfo = pMsg->rpcMsg.pCont;
+static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) {
+  SMnode        *pMnode = pReq->pMnode;
+  STableInfoReq *pInfo = pReq->rpcMsg.pCont;
 
   mDebug("stb:%s, start to retrieve meta", pInfo->tableFname);
 
@@ -786,8 +735,8 @@ static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg) {
   mndReleaseDb(pMnode, pDb);
   mndReleaseStb(pMnode, pStb);
 
-  pMsg->pCont = pMeta;
-  pMsg->contLen = contLen;
+  pReq->pCont = pMeta;
+  pReq->contLen = contLen;
 
   mDebug("stb:%s, meta is retrieved, cols:%d tags:%d", pInfo->tableFname, pStb->numOfColumns, pStb->numOfTags);
   return 0;
@@ -820,8 +769,8 @@ static int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs
   return 0;
 }
 
-static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) {
-  SMnode *pMnode = pMsg->pMnode;
+static int32_t mndGetStbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) {
+  SMnode *pMnode = pReq->pMnode;
   SSdb   *pSdb = pMnode->pSdb;
 
   if (mndGetNumOfStbs(pMnode, pShow->db, &pShow->numOfRows) != 0) {
@@ -883,8 +832,8 @@ static void mndExtractTableName(char *tableId, char *name) {
   }
 }
 
-static int32_t mndRetrieveStb(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) {
-  SMnode  *pMnode = pMsg->pMnode;
+static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) {
+  SMnode  *pMnode = pReq->pMnode;
   SSdb    *pSdb = pMnode->pSdb;
   int32_t  numOfRows = 0;
   SStbObj *pStb = NULL;

source/dnode/mnode/impl/src/mndTopic.c

@@ -74,6 +74,15 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
   SDB_SET_INT32(pRaw, dataPos, pTopic->version, TOPIC_ENCODE_OVER);
   SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER);
   SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_ENCODE_OVER);
+  int32_t logicalPlanLen = strlen(pTopic->logicalPlan) + 1;
+  SDB_SET_INT32(pRaw, dataPos, strlen(pTopic->logicalPlan)+1, TOPIC_ENCODE_OVER);
+  SDB_SET_BINARY(pRaw, dataPos, pTopic->logicalPlan, logicalPlanLen, TOPIC_ENCODE_OVER);
+
+  int32_t physicalPlanLen = strlen(pTopic->physicalPlan) + 1;
+  pTopic->physicalPlan = calloc(physicalPlanLen, sizeof(char));
+  if (pTopic->physicalPlan == NULL) goto TOPIC_ENCODE_OVER;
+  SDB_SET_INT32(pRaw, dataPos, strlen(pTopic->physicalPlan)+1, TOPIC_ENCODE_OVER);
+  SDB_SET_BINARY(pRaw, dataPos, pTopic->physicalPlan, physicalPlanLen, TOPIC_ENCODE_OVER);
 
   SDB_SET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_ENCODE_OVER);
   SDB_SET_DATALEN(pRaw, dataPos, TOPIC_ENCODE_OVER);
@@ -83,6 +92,12 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
 TOPIC_ENCODE_OVER:
   if (terrno != TSDB_CODE_SUCCESS) {
     mError("topic:%s, failed to encode to raw:%p since %s", pTopic->name, pRaw, terrstr());
+    /*if (pTopic->logicalPlan) {*/
+      /*free(pTopic->logicalPlan);*/
+    /*}*/
+    /*if (pTopic->physicalPlan) {*/
+      /*free(pTopic->physicalPlan);*/
+    /*}*/
     sdbFreeRaw(pRaw);
     return NULL;
   }
@@ -118,11 +133,26 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
   SDB_GET_INT64(pRaw, dataPos, &pTopic->dbUid, TOPIC_DECODE_OVER);
   SDB_GET_INT32(pRaw, dataPos, &pTopic->version, TOPIC_DECODE_OVER);
   SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER);
+
+  pTopic->sql = calloc(pTopic->sqlLen + 1, sizeof(char));
   SDB_GET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_DECODE_OVER);
+
   SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER);
-  SDB_GET_BINARY(pRaw, dataPos, pTopic->logicalPlan, len, TOPIC_DECODE_OVER);
+  pTopic->logicalPlan = calloc(len+1, sizeof(char));
+  if (pTopic->logicalPlan == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto TOPIC_DECODE_OVER;
+  }
+  SDB_GET_BINARY(pRaw, dataPos, pTopic->logicalPlan, len+1, TOPIC_DECODE_OVER);
+
   SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER);
-  SDB_GET_BINARY(pRaw, dataPos, pTopic->physicalPlan, len, TOPIC_DECODE_OVER);
+  pTopic->logicalPlan = calloc(len + 1, sizeof(char));
+  if (pTopic->physicalPlan == NULL) {
+    free(pTopic->logicalPlan);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto TOPIC_DECODE_OVER;
+  }
+  SDB_GET_BINARY(pRaw, dataPos, pTopic->physicalPlan, len+1, TOPIC_DECODE_OVER);
 
   SDB_GET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_DECODE_OVER)
 
@@ -178,7 +208,7 @@ void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic) {
 
 static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) {
   SName name = {0};
-  tNameFromString(&name, topicName, T_NAME_ACCT | T_NAME_DB | T_NAME_TOPIC);
+  tNameFromString(&name, topicName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
 
   char db[TSDB_TABLE_FNAME_LEN] = {0};
   tNameGetFullDbName(&name, db);
@@ -203,20 +233,24 @@ static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMq
   return pDrop;
 }
 
-static int32_t mndCheckCreateTopicMsg(SCMCreateTopicReq *pCreate) {
+static int32_t mndCheckCreateTopicMsg(SCMCreateTopicReq *creattopReq) {
   // deserialize and other stuff
   return 0;
 }
 
 static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq *pCreate, SDbObj *pDb) {
   SMqTopicObj topicObj = {0};
-  tstrncpy(topicObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN);
+  tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN);
   tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN);
   topicObj.createTime = taosGetTimestampMs();
   topicObj.updateTime = topicObj.createTime;
   topicObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
   topicObj.dbUid = pDb->uid;
   topicObj.version = 1;
+  topicObj.sql = strdup(pCreate->sql);
+  topicObj.physicalPlan = strdup(pCreate->physicalPlan);
+  topicObj.logicalPlan = strdup(pCreate->logicalPlan);
+  topicObj.sqlLen = strlen(pCreate->sql);
 
   SSdbRaw *pTopicRaw = mndTopicActionEncode(&topicObj);
   if (pTopicRaw == NULL) return -1;
@@ -228,46 +262,47 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq
 static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg) {
   SMnode            *pMnode = pMsg->pMnode;
   char              *msgStr = pMsg->rpcMsg.pCont;
-  SCMCreateTopicReq *pCreate;
-  tDeserializeSCMCreateTopicReq(msgStr, pCreate);
 
-  mDebug("topic:%s, start to create", pCreate->name);
+  SCMCreateTopicReq createTopicReq = {0};
+  tDeserializeSCMCreateTopicReq(msgStr, &createTopicReq);
+
+  mDebug("topic:%s, start to create, sql:%s", createTopicReq.name, createTopicReq.sql);
 
-  if (mndCheckCreateTopicMsg(pCreate) != 0) {
-    mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
+  if (mndCheckCreateTopicMsg(&createTopicReq) != 0) {
+    mError("topic:%s, failed to create since %s", createTopicReq.name, terrstr());
     return -1;
   }
 
-  SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pCreate->name);
+  SMqTopicObj *pTopic = mndAcquireTopic(pMnode, createTopicReq.name);
   if (pTopic != NULL) {
     sdbRelease(pMnode->pSdb, pTopic);
-    if (pCreate->igExists) {
-      mDebug("topic:%s, already exist, ignore exist is set", pCreate->name);
+    if (createTopicReq.igExists) {
+      mDebug("topic:%s, already exist, ignore exist is set", createTopicReq.name);
       return 0;
     } else {
       terrno = TSDB_CODE_MND_TOPIC_ALREADY_EXIST;
-      mError("db:%s, failed to create since %s", pCreate->name, terrstr());
+      mError("db:%s, failed to create since %s", createTopicReq.name, terrstr());
       return -1;
     }
   }
 
-  SDbObj *pDb = mndAcquireDbByTopic(pMnode, pCreate->name);
+  SDbObj *pDb = mndAcquireDbByTopic(pMnode, createTopicReq.name);
   if (pDb == NULL) {
     terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
-    mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
+    mError("topic:%s, failed to create since %s", createTopicReq.name, terrstr());
     return -1;
   }
 
-  int32_t code = mndCreateTopic(pMnode, pMsg, pCreate, pDb);
+  int32_t code = mndCreateTopic(pMnode, pMsg, &createTopicReq, pDb);
   mndReleaseDb(pMnode, pDb);
 
   if (code != 0) {
     terrno = code;
-    mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
+    mError("topic:%s, failed to create since %s", createTopicReq.name, terrstr());
     return -1;
   }
 
-  return TSDB_CODE_MND_ACTION_IN_PROGRESS;
+  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pMsg, SMqTopicObj *pTopic) { return 0; }

source/dnode/mnode/impl/test/profile/profile.cpp

@@ -96,6 +96,38 @@ TEST_F(MndTestProfile, 03_ConnectMsg_Show) {
 }
 
 TEST_F(MndTestProfile, 04_HeartBeatMsg) {
+  SClientHbBatchReq batchReq;
+  batchReq.reqs = taosArrayInit(0, sizeof(SClientHbReq));
+  SClientHbReq req = {0};
+  req.connKey = {.connId = 123, .hbType = HEARTBEAT_TYPE_MQ};
+  req.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
+  SKv kv;
+  kv.key = (void*)"abc";
+  kv.keyLen = 4;
+  kv.value = (void*)"bcd";
+  kv.valueLen = 4;
+  taosHashPut(req.info, kv.key, kv.keyLen, kv.value, kv.valueLen);
+  taosArrayPush(batchReq.reqs, &req);
+
+  int32_t tlen = tSerializeSClientHbBatchReq(NULL, &batchReq);
+  
+  void* buf = (SClientHbBatchReq*)rpcMallocCont(tlen);
+  void* bufCopy = buf;
+  tSerializeSClientHbBatchReq(&bufCopy, &batchReq);
+  SRpcMsg* pMsg = test.SendReq(TDMT_MND_HEARTBEAT, buf, tlen);
+  ASSERT_NE(pMsg, nullptr);
+  ASSERT_EQ(pMsg->code, 0);
+  char* pRspChar = (char*)pMsg->pCont;
+  SClientHbBatchRsp rsp = {0};
+  tDeserializeSClientHbBatchRsp(pRspChar, &rsp);
+  int sz = taosArrayGetSize(rsp.rsps);
+  ASSERT_EQ(sz, 0);
+  //SClientHbRsp* pRsp = (SClientHbRsp*) taosArrayGet(rsp.rsps, 0);
+  //EXPECT_EQ(pRsp->connKey.connId, 123);
+  //EXPECT_EQ(pRsp->connKey.hbType, HEARTBEAT_TYPE_MQ);
+  //EXPECT_EQ(pRsp->status, 0);
+
+#if 0
   int32_t contLen = sizeof(SHeartBeatReq);
 
   SHeartBeatReq* pReq = (SHeartBeatReq*)rpcMallocCont(contLen);
@@ -129,9 +161,12 @@ TEST_F(MndTestProfile, 04_HeartBeatMsg) {
   EXPECT_EQ(pRsp->epSet.numOfEps, 1);
   EXPECT_EQ(pRsp->epSet.port[0], 9031);
   EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost");
+#endif
 }
 
 TEST_F(MndTestProfile, 05_KillConnMsg) {
+  // temporary remove since kill will use new heartbeat msg
+#if 0
   {
     int32_t contLen = sizeof(SKillConnReq);
 
@@ -190,6 +225,7 @@ TEST_F(MndTestProfile, 05_KillConnMsg) {
 
     connId = pRsp->connId;
   }
+#endif
 }
 
 TEST_F(MndTestProfile, 06_KillConnMsg_InvalidConn) {
@@ -204,6 +240,8 @@ TEST_F(MndTestProfile, 06_KillConnMsg_InvalidConn) {
 }
 
 TEST_F(MndTestProfile, 07_KillQueryMsg) {
+  // temporary remove since kill will use new heartbeat msg
+#if 0
   {
     int32_t contLen = sizeof(SKillQueryReq);
 
@@ -252,6 +290,7 @@ TEST_F(MndTestProfile, 07_KillQueryMsg) {
     EXPECT_EQ(pRsp->epSet.port[0], 9031);
     EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost");
   }
+#endif
 }
 
 TEST_F(MndTestProfile, 08_KillQueryMsg_InvalidConn) {

source/dnode/mnode/impl/test/stb/CMakeLists.txt

 aux_source_directory(. STB_SRC)
-add_executable(dnode_test_stb ${STB_SRC})
+add_executable(mnode_test_stb ${STB_SRC})
 target_link_libraries(
-    dnode_test_stb
+    mnode_test_stb
     PUBLIC sut
 )
 
 add_test(
-    NAME dnode_test_stb
-    COMMAND dnode_test_stb
+    NAME mnode_test_stb
+    COMMAND mnode_test_stb
 )

source/dnode/vnode/CMakeLists.txt

-add_subdirectory(meta)
-add_subdirectory(tq)
-add_subdirectory(tsdb)
-add_subdirectory(impl)
+aux_source_directory(src/meta META_SRC)
+aux_source_directory(src/tq TQ_SRC)
+aux_source_directory(src/tsdb TSDB_SRC)
+aux_source_directory(src/vnd VND_SRC)
\ No newline at end of file
+list(APPEND
+    VNODE_SRC
+    ${META_SRC}
+    ${TQ_SRC}
+    ${TSDB_SRC}
+    ${VND_SRC}
+)
+
+add_library(vnode STATIC ${VNODE_SRC})
+target_include_directories(
+    vnode
+    PUBLIC inc
+    PRIVATE src/inc
+)
+target_link_libraries(
+    vnode
+    PUBLIC os
+    PUBLIC util
+    PUBLIC common
+    PUBLIC transport
+    PUBLIC bdb
+    PUBLIC tfs
+    PUBLIC wal
+    PUBLIC qworker
+)
+
+if(${BUILD_TEST})
+    # add_subdirectory(test)
+endif(${BUILD_TEST})

source/dnode/vnode/impl/CMakeLists.txt

-aux_source_directory(src VNODE_SRC)
-add_library(vnode STATIC ${VNODE_SRC})
-target_include_directories(
-    vnode
-    PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/vnode"
-    private "${CMAKE_CURRENT_SOURCE_DIR}/inc"
-)
-target_link_libraries(
-    vnode
-    PUBLIC os
-    PUBLIC transport
-    PUBLIC meta
-    PUBLIC tq
-    PUBLIC tsdb
-    PUBLIC wal
-    PUBLIC sync
-    PUBLIC cjson
-    PUBLIC qworker
-)
-
-# test
-if(${BUILD_TEST})
-#    add_subdirectory(test)
-endif(${BUILD_TEST})

source/dnode/vnode/impl/inc/vnodeBufferPool.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_VNODE_BUFFER_POOL_H_
-#define _TD_VNODE_BUFFER_POOL_H_
-
-#include "tlist.h"
-#include "vnode.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct SVBufPool SVBufPool;
-
-int   vnodeOpenBufPool(SVnode *pVnode);
-void  vnodeCloseBufPool(SVnode *pVnode);
-int   vnodeBufPoolSwitch(SVnode *pVnode);
-int   vnodeBufPoolRecycle(SVnode *pVnode);
-void *vnodeMalloc(SVnode *pVnode, uint64_t size);
-bool  vnodeBufPoolIsFull(SVnode *pVnode);
-
-SMemAllocatorFactory *vBufPoolGetMAF(SVnode *pVnode);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_VNODE_BUFFER_POOL_H_*/
\ No newline at end of file

source/dnode/vnode/impl/inc/vnodeCfg.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_VNODE_CFG_H_
-#define _TD_VNODE_CFG_H_
-
-#include "vnode.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-extern const SVnodeCfg defaultVnodeOptions;
-
-int  vnodeValidateOptions(const SVnodeCfg *);
-void vnodeOptionsCopy(SVnodeCfg *pDest, const SVnodeCfg *pSrc);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_VNODE_CFG_H_*/
\ No newline at end of file

source/dnode/vnode/impl/inc/vnodeCommit.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_VNODE_COMMIT_H_
-#define _TD_VNODE_COMMIT_H_
-
-#include "vnode.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define vnodeShouldCommit vnodeBufPoolIsFull
-int vnodeSyncCommit(SVnode *pVnode);
-int vnodeAsyncCommit(SVnode *pVnode);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_VNODE_COMMIT_H_*/
\ No newline at end of file

source/dnode/vnode/impl/inc/vnodeDef.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_VNODE_DEF_H_
-#define _TD_VNODE_DEF_H_
-
-#include "mallocator.h"
-#include "sync.h"
-#include "tcoding.h"
-#include "tlist.h"
-#include "tlockfree.h"
-#include "tmacro.h"
-#include "wal.h"
-#include "tfs.h"
-
-#include "vnode.h"
-
-#include "vnodeBufferPool.h"
-#include "vnodeCfg.h"
-#include "vnodeCommit.h"
-#include "vnodeFS.h"
-#include "vnodeMemAllocator.h"
-#include "vnodeQuery.h"
-#include "vnodeRequest.h"
-#include "vnodeStateMgr.h"
-#include "vnodeSync.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct SVnodeTask {
-  TD_DLIST_NODE(SVnodeTask);
-  void* arg;
-  int (*execute)(void*);
-} SVnodeTask;
-
-typedef struct SVnodeMgr {
-  td_mode_flag_t vnodeInitFlag;
-  // For commit
-  bool            stop;
-  uint16_t        nthreads;
-  pthread_t*      threads;
-  pthread_mutex_t mutex;
-  pthread_cond_t  hasTask;
-  TD_DLIST(SVnodeTask) queue;
-  // For vnode Mgmt
-  SDnode*           pDnode;
-  PutReqToVQueryQFp putReqToVQueryQFp;
-} SVnodeMgr;
-
-extern SVnodeMgr vnodeMgr;
-
-struct SVnode {
-  int32_t     vgId;
-  char*       path;
-  SVnodeCfg   config;
-  SVState     state;
-  SVBufPool*  pBufPool;
-  SMeta*      pMeta;
-  STsdb*      pTsdb;
-  STQ*        pTq;
-  SWal*       pWal;
-  SVnodeSync* pSync;
-  SVnodeFS*   pFs;
-  tsem_t      canCommit;
-  SQHandle*   pQuery;
-  SDnode*     pDnode;
-};
-
-int vnodeScheduleTask(SVnodeTask* task);
-
-int32_t vnodePutReqToVQueryQ(SVnode *pVnode, struct SRpcMsg *pReq);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_VNODE_DEF_H_*/
\ No newline at end of file

source/dnode/vnode/impl/inc/vnodeInt.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_VNODE_INT_H_
-#define _TD_VNODE_INT_H_
-
-#include "vnode.h"
-
-#include "meta.h"
-#include "sync.h"
-#include "tlog.h"
-#include "tq.h"
-#include "tsdb.h"
-#include "wal.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-extern int32_t vDebugFlag;
-
-#define vFatal(...) { if (vDebugFlag & DEBUG_FATAL) { taosPrintLog("VND FATAL ", 255, __VA_ARGS__); }}
-#define vError(...) { if (vDebugFlag & DEBUG_ERROR) { taosPrintLog("VND ERROR ", 255, __VA_ARGS__); }}
-#define vWarn(...)  { if (vDebugFlag & DEBUG_WARN)  { taosPrintLog("VND WARN ", 255, __VA_ARGS__); }}
-#define vInfo(...)  { if (vDebugFlag & DEBUG_INFO)  { taosPrintLog("VND ", 255, __VA_ARGS__); }}
-#define vDebug(...) { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("VND ", vDebugFlag, __VA_ARGS__); }}
-#define vTrace(...) { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("VND ", vDebugFlag, __VA_ARGS__); }}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_VNODE_INT_H_*/

source/dnode/vnode/impl/inc/vnodeMAF.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_VNODE_MAF_H_
-#define _TD_VNODE_MAF_H_
-
-#include "vnode.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-int  vnodeOpenMAF(SVnode *pVnode);
-void vnodeCloseMAF(SVnode *pVnode);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_VNODE_MAF_H_*/
\ No newline at end of file

source/dnode/vnode/impl/inc/vnodeMemAllocator.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_VNODE_MEM_ALLOCATOR_H_
-#define _TD_VNODE_MEM_ALLOCATOR_H_
-
-#include "os.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct SVArenaNode {
-  TD_SLIST_NODE(SVArenaNode);
-  uint64_t size;  // current node size
-  void *   ptr;
-  char     data[];
-} SVArenaNode;
-
-typedef struct SVMemAllocator {
-  T_REF_DECLARE()
-  TD_DLIST_NODE(SVMemAllocator);
-  uint64_t     capacity;
-  uint64_t     ssize;
-  uint64_t     lsize;
-  SVArenaNode *pNode;
-  TD_SLIST(SVArenaNode) nlist;
-} SVMemAllocator;
-
-SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize);
-void            vmaDestroy(SVMemAllocator *pVMA);
-void            vmaReset(SVMemAllocator *pVMA);
-void *          vmaMalloc(SVMemAllocator *pVMA, uint64_t size);
-void            vmaFree(SVMemAllocator *pVMA, void *ptr);
-bool            vmaIsFull(SVMemAllocator *pVMA);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_VNODE_MEM_ALLOCATOR_H_*/

source/dnode/vnode/impl/inc/vnodeRead.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_VNODE_READ_H_
-#define _TD_VNODE_READ_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-#include "vnodeInt.h"
-
-void vnodeProcessReadMsg(SVnode *pVnode, SVnodeMsg *pMsg);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_VNODE_READ_H_*/

source/dnode/vnode/impl/inc/vnodeRequest.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_VNODE_REQUEST_H_
-#define _TD_VNODE_REQUEST_H_
-
-#include "vnode.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-// SVDropTbReq
-// int   vnodeBuildDropTableReq(void **buf, const SVDropTbReq *pReq);
-// void *vnodeParseDropTableReq(void *buf, SVDropTbReq *pReq);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_VNODE_REQUEST_H_*/
\ No newline at end of file

source/dnode/vnode/impl/inc/vnodeWrite.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_VNODE_WRITE_H_
-#define _TD_VNODE_WRITE_H_
-
-#include "vnode.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-void vnodeProcessWriteMsg(SVnode* pVnode, SVnodeMsg* pMsg);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_VNODE_WRITE_H_*/

source/dnode/vnode/impl/src/vnodeFS.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "vnodeDef.h"
\ No newline at end of file

source/dnode/vnode/impl/src/vnodeRead.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "vnodeDef.h"
\ No newline at end of file

source/dnode/vnode/impl/src/vnodeRequest.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "vnodeDef.h"
-
-#if 0
-
-static int   vnodeBuildCreateTableReq(void **buf, const SVCreateTableReq *pReq);
-static void *vnodeParseCreateTableReq(void *buf, SVCreateTableReq *pReq);
-
-int vnodeBuildReq(void **buf, const SVnodeReq *pReq, tmsg_t type) {
-  int tsize = 0;
-
-  tsize += taosEncodeFixedU64(buf, pReq->ver);
-  switch (type) {
-    case TDMT_VND_CREATE_STB:
-      tsize += vnodeBuildCreateTableReq(buf, &(pReq->ctReq));
-      break;
-    case TDMT_VND_SUBMIT:
-      /* code */
-      break;
-    default:
-      break;
-  }
-  /* TODO */
-  return tsize;
-}
-
-void *vnodeParseReq(void *buf, SVnodeReq *pReq, tmsg_t type) {
-  buf = taosDecodeFixedU64(buf, &(pReq->ver));
-
-  switch (type) {
-    case TDMT_VND_CREATE_STB:
-      buf = vnodeParseCreateTableReq(buf, &(pReq->ctReq));
-      break;
-
-    default:
-      break;
-  }
-
-  // TODO
-  return buf;
-}
-
-static int vnodeBuildCreateTableReq(void **buf, const SVCreateTableReq *pReq) {
-  int tsize = 0;
-
-  tsize += taosEncodeString(buf, pReq->name);
-  tsize += taosEncodeFixedU32(buf, pReq->ttl);
-  tsize += taosEncodeFixedU32(buf, pReq->keep);
-  tsize += taosEncodeFixedU8(buf, pReq->type);
-
-  switch (pReq->type) {
-    case META_SUPER_TABLE:
-      tsize += taosEncodeFixedU64(buf, pReq->stbCfg.suid);
-      tsize += tdEncodeSchema(buf, pReq->stbCfg.pSchema);
-      tsize += tdEncodeSchema(buf, pReq->stbCfg.pTagSchema);
-      break;
-    case META_CHILD_TABLE:
-      tsize += taosEncodeFixedU64(buf, pReq->ctbCfg.suid);
-      tsize += tdEncodeKVRow(buf, pReq->ctbCfg.pTag);
-      break;
-    case META_NORMAL_TABLE:
-      tsize += tdEncodeSchema(buf, pReq->ntbCfg.pSchema);
-      break;
-    default:
-      break;
-  }
-
-  return tsize;
-}
-
-static void *vnodeParseCreateTableReq(void *buf, SVCreateTableReq *pReq) {
-  buf = taosDecodeString(buf, &(pReq->name));
-  buf = taosDecodeFixedU32(buf, &(pReq->ttl));
-  buf = taosDecodeFixedU32(buf, &(pReq->keep));
-  buf = taosDecodeFixedU8(buf, &(pReq->type));
-
-  switch (pReq->type) {
-    case META_SUPER_TABLE:
-      buf = taosDecodeFixedU64(buf, &(pReq->stbCfg.suid));
-      buf = tdDecodeSchema(buf, &(pReq->stbCfg.pSchema));
-      buf = tdDecodeSchema(buf, &(pReq->stbCfg.pTagSchema));
-      break;
-    case META_CHILD_TABLE:
-      buf = taosDecodeFixedU64(buf, &(pReq->ctbCfg.suid));
-      buf = tdDecodeKVRow(buf, &(pReq->ctbCfg.pTag));
-      break;
-    case META_NORMAL_TABLE:
-      buf = tdDecodeSchema(buf, &(pReq->ntbCfg.pSchema));
-      break;
-    default:
-      break;
-  }
-
-  return buf;
-}
-
-int vnodeBuildDropTableReq(void **buf, const SVDropTbReq *pReq) {
-  // TODO
-  return 0;
-}
-
-void *vnodeParseDropTableReq(void *buf, SVDropTbReq *pReq) {
-  // TODO
-}
-#endif
\ No newline at end of file

source/dnode/vnode/impl/src/vnodeSync.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file

source/dnode/vnode/impl/test/CMakeLists.txt

-# Vnode API test
-add_executable(vnodeApiTests "")
-target_sources(vnodeApiTests
-    PRIVATE
-    "vnodeApiTests.cpp"
-)
-target_link_libraries(vnodeApiTests vnode gtest gtest_main)
-
-add_test(
-    NAME vnode_api_tests 
-    COMMAND ${CMAKE_CURRENT_BINARY_DIR}/vnodeApiTests
-    )
\ No newline at end of file

source/dnode/vnode/impl/test/vBenchmarkTest.cpp

-// https://stackoverflow.com/questions/8565666/benchmarking-with-googletest
-// https://github.com/google/benchmark
\ No newline at end of file

source/dnode/vnode/impl/test/vnodeApiTests.cpp

-/**
- * @file vnodeApiTests.cpp
- * @author hzcheng (hzcheng@taosdata.com)
- * @brief VNODE module API tests
- * @version 0.1
- * @date 2021-12-13
- *
- * @copyright Copyright (c) 2021
- *
- */
-
-#include <gtest/gtest.h>
-#include <iostream>
-
-#include "vnode.h"
-
-static STSchema *vtCreateBasicSchema() {
-  STSchemaBuilder sb;
-  STSchema *      pSchema = NULL;
-
-  tdInitTSchemaBuilder(&sb, 0);
-
-  tdAddColToSchema(&sb, TSDB_DATA_TYPE_TIMESTAMP, 0, 0);
-  for (int i = 1; i < 10; i++) {
-    tdAddColToSchema(&sb, TSDB_DATA_TYPE_INT, i, 0);
-  }
-
-  pSchema = tdGetSchemaFromBuilder(&sb);
-
-  tdDestroyTSchemaBuilder(&sb);
-
-  return pSchema;
-}
-
-static STSchema *vtCreateBasicTagSchema() {
-  STSchemaBuilder sb;
-  STSchema *      pSchema = NULL;
-
-  tdInitTSchemaBuilder(&sb, 0);
-
-  tdAddColToSchema(&sb, TSDB_DATA_TYPE_TIMESTAMP, 0, 0);
-  for (int i = 10; i < 12; i++) {
-    tdAddColToSchema(&sb, TSDB_DATA_TYPE_BINARY, i, 20);
-  }
-
-  pSchema = tdGetSchemaFromBuilder(&sb);
-
-  tdDestroyTSchemaBuilder(&sb);
-
-  return pSchema;
-}
-
-static SKVRow vtCreateBasicTag() {
-  SKVRowBuilder rb;
-  SKVRow        pTag;
-
-  tdInitKVRowBuilder(&rb);
-
-  for (int i = 0; i < 2; i++) {
-    void *pVal = malloc(sizeof(VarDataLenT) + strlen("foo"));
-    varDataLen(pVal) = strlen("foo");
-    memcpy(varDataVal(pVal), "foo", strlen("foo"));
-
-    tdAddColToKVRow(&rb, i, TSDB_DATA_TYPE_BINARY, pVal);
-    free(pVal);
-  }
-
-  pTag = tdGetKVRowFromBuilder(&rb);
-  tdDestroyKVRowBuilder(&rb);
-
-  return pTag;
-}
-
-static void vtBuildCreateStbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) {
-  SRpcMsg * pMsg;
-  STSchema *pSchema;
-  STSchema *pTagSchema;
-  int       zs;
-  void *    pBuf;
-
-  pSchema = vtCreateBasicSchema();
-  pTagSchema = vtCreateBasicTagSchema();
-
-  SVnodeReq vCreateSTbReq;
-  vnodeSetCreateStbReq(&vCreateSTbReq, tbname, UINT32_MAX, UINT32_MAX, suid, pSchema, pTagSchema);
-
-  zs = vnodeBuildReq(NULL, &vCreateSTbReq, TDMT_VND_CREATE_STB);
-  pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + zs);
-  pMsg->msgType = TDMT_VND_CREATE_STB;
-  pMsg->contLen = zs;
-  pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(SRpcMsg));
-
-  pBuf = pMsg->pCont;
-  vnodeBuildReq(&pBuf, &vCreateSTbReq, TDMT_VND_CREATE_STB);
-  META_CLEAR_TB_CFG(&vCreateSTbReq);
-
-  tdFreeSchema(pSchema);
-  tdFreeSchema(pTagSchema);
-
-  *ppMsg = pMsg;
-}
-
-static void vtBuildCreateCtbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) {
-  SRpcMsg *pMsg;
-  int      tz;
-  SKVRow   pTag = vtCreateBasicTag();
-
-  SVnodeReq vCreateCTbReq;
-  vnodeSetCreateCtbReq(&vCreateCTbReq, tbname, UINT32_MAX, UINT32_MAX, suid, pTag);
-
-  tz = vnodeBuildReq(NULL, &vCreateCTbReq, TDMT_VND_CREATE_TABLE);
-  pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + tz);
-  pMsg->msgType = TDMT_VND_CREATE_TABLE;
-  pMsg->contLen = tz;
-  pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg));
-  void *pBuf = pMsg->pCont;
-
-  vnodeBuildReq(&pBuf, &vCreateCTbReq, TDMT_VND_CREATE_TABLE);
-  META_CLEAR_TB_CFG(&vCreateCTbReq);
-  free(pTag);
-
-  *ppMsg = pMsg;
-}
-
-static void vtBuildCreateNtbReq(char *tbname, SRpcMsg **ppMsg) {
-  // TODO
-}
-
-static void vtBuildSubmitReq(SRpcMsg **ppMsg) {
-  SRpcMsg *   pMsg;
-  SSubmitMsg *pSubmitMsg;
-  SSubmitBlk *pSubmitBlk;
-  int         tz = 1024;  // TODO
-
-  pMsg = (SRpcMsg *)malloc(sizeof(*pMsg) + tz);
-  pMsg->msgType = TDMT_VND_SUBMIT;
-  pMsg->contLen = tz;
-  pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg));
-
-  // For submit msg header
-  pSubmitMsg = (SSubmitMsg *)(pMsg->pCont);
-  // pSubmitMsg->header.contLen = 0;
-  // pSubmitMsg->header.vgId = 0;
-  // pSubmitMsg->length = 0;
-  pSubmitMsg->numOfBlocks = 1;
-
-  // For submit blk
-  pSubmitBlk = (SSubmitBlk *)(pSubmitMsg->blocks);
-  pSubmitBlk->uid = 0;
-  pSubmitBlk->tid = 0;
-  pSubmitBlk->padding = 0;
-  pSubmitBlk->sversion = 0;
-  pSubmitBlk->dataLen = 0;
-  pSubmitBlk->numOfRows = 0;
-
-  // For row batch
-
-  *ppMsg = pMsg;
-}
-
-static void vtClearMsgBatch(SArray *pMsgArr) {
-  SRpcMsg *pMsg;
-  for (size_t i = 0; i < taosArrayGetSize(pMsgArr); i++) {
-    pMsg = *(SRpcMsg **)taosArrayGet(pMsgArr, i);
-    free(pMsg);
-  }
-
-  taosArrayClear(pMsgArr);
-}
-
-static void vtProcessAndApplyReqs(SVnode *pVnode, SArray *pMsgArr) {
-  int      rcode;
-  SRpcMsg *pReq;
-  SRpcMsg *pRsp;
-
-  rcode = vnodeProcessWMsgs(pVnode, pMsgArr);
-  GTEST_ASSERT_EQ(rcode, 0);
-
-  for (size_t i = 0; i < taosArrayGetSize(pMsgArr); i++) {
-    pReq = *(SRpcMsg **)taosArrayGet(pMsgArr, i);
-    rcode = vnodeApplyWMsg(pVnode, pReq, NULL);
-    GTEST_ASSERT_EQ(rcode, 0);
-  }
-}
-
-TEST(vnodeApiTest, vnode_simple_create_table_test) {
-  tb_uid_t suid = 1638166374163;
-  SRpcMsg *pMsg;
-  SArray * pMsgArr = NULL;
-  SVnode * pVnode;
-  int      rcode;
-  int      ntables = 1000000;
-  int      batch = 10;
-  char     tbname[128];
-
-  pMsgArr = (SArray *)taosArrayInit(batch, sizeof(pMsg));
-
-  vnodeDestroy("vnode1");
-  GTEST_ASSERT_GE(vnodeInit(2), 0);
-
-  // CREATE AND OPEN A VNODE
-  pVnode = vnodeOpen("vnode1", NULL);
-  ASSERT_NE(pVnode, nullptr);
-
-  // CREATE A SUPER TABLE
-  sprintf(tbname, "st");
-  vtBuildCreateStbReq(suid, tbname, &pMsg);
-  taosArrayPush(pMsgArr, &pMsg);
-  vtProcessAndApplyReqs(pVnode, pMsgArr);
-  vtClearMsgBatch(pMsgArr);
-
-  // CREATE A LOT OF CHILD TABLES
-  for (int i = 0; i < ntables / batch; i++) {
-    // Build request batch
-    for (int j = 0; j < batch; j++) {
-      sprintf(tbname, "ct%d", i * batch + j + 1);
-      vtBuildCreateCtbReq(suid, tbname, &pMsg);
-      taosArrayPush(pMsgArr, &pMsg);
-    }
-
-    // Process request batch
-    vtProcessAndApplyReqs(pVnode, pMsgArr);
-
-    // Clear request batch
-    vtClearMsgBatch(pMsgArr);
-  }
-
-  // CLOSE THE VNODE
-  vnodeClose(pVnode);
-  vnodeCleanup();
-
-  taosArrayDestroy(pMsgArr);
-}
-
-TEST(vnodeApiTest, vnode_simple_insert_test) {
-  const char *vname = "vnode2";
-  char        tbname[128];
-  tb_uid_t    suid = 1638166374163;
-  SRpcMsg *   pMsg;
-  SArray *    pMsgArr;
-  int         rcode;
-  SVnode *    pVnode;
-  int         batch = 1;
-  int         loop = 1000000;
-
-  pMsgArr = (SArray *)taosArrayInit(0, sizeof(pMsg));
-
-  vnodeDestroy(vname);
-
-  GTEST_ASSERT_GE(vnodeInit(2), 0);
-
-  // Open a vnode
-  pVnode = vnodeOpen(vname, NULL);
-  GTEST_ASSERT_NE(pVnode, nullptr);
-
-  // 1. CREATE A SUPER TABLE
-  sprintf(tbname, "st");
-  vtBuildCreateStbReq(suid, tbname, &pMsg);
-  taosArrayPush(pMsgArr, &pMsg);
-  vtProcessAndApplyReqs(pVnode, pMsgArr);
-  vtClearMsgBatch(pMsgArr);
-
-  // 2. CREATE A CHILD TABLE
-  sprintf(tbname, "t0");
-  vtBuildCreateCtbReq(suid, tbname, &pMsg);
-  taosArrayPush(pMsgArr, &pMsg);
-  vtProcessAndApplyReqs(pVnode, pMsgArr);
-  vtClearMsgBatch(pMsgArr);
-
-  // 3. WRITE A LOT OF TIME-SERIES DATA
-  for (int j = 0; j < loop; j++) {
-    for (int i = 0; i < batch; i++) {
-      vtBuildSubmitReq(&pMsg);
-      taosArrayPush(pMsgArr, &pMsg);
-    }
-    vtProcessAndApplyReqs(pVnode, pMsgArr);
-    vtClearMsgBatch(pMsgArr);
-  }
-
-  // Close the vnode
-  vnodeClose(pVnode);
-  vnodeCleanup();
-
-  taosArrayDestroy(pMsgArr);
-}
\ No newline at end of file

include/dnode/vnode/tq/tq.h → source/dnode/vnode/inc/tq.h

@@ -25,6 +25,7 @@
 #include "trpc.h"
 #include "ttimer.h"
 #include "tutil.h"
+#include "meta.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -314,6 +315,26 @@ const void* tqDeserializeGroup(const STqSerializedHead*, STqGroup**);
 
 static int tqQueryExecuting(int32_t status) { return status; }
 
+typedef struct STqReadHandle {
+  int64_t ver;
+  SSubmitMsg* pMsg;
+  SSubmitBlk* pBlock;
+  SSubmitMsgIter msgIter;
+  SSubmitBlkIter blkIter;
+  SMeta* pMeta;
+} STqReadHandle;
+
+typedef struct SSubmitBlkScanInfo {
+  
+} SSubmitBlkScanInfo;
+
+STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta, SSubmitMsg *pMsg);
+bool tqNextDataBlock(STqReadHandle* pHandle);
+int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo *pBlockInfo);
+//return SArray<SColumnInfoData>
+SArray *tqRetrieveDataBlock(STqReadHandle* pHandle, SArray* pColumnIdList);
+//int tqLoadDataBlock(SExecTaskInfo* pTaskInfo, SSubmitBlkScanInfo* pSubmitBlkScanInfo, SSDataBlock* pBlock, uint32_t status);
+
 #ifdef __cplusplus
 }
 #endif

include/dnode/vnode/vnode.h → source/dnode/vnode/inc/vnode.h

@@ -36,45 +36,25 @@ typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq);
 
 typedef struct SVnodeCfg {
   int32_t  vgId;
-  SDnode *pDnode;
-
-  /** vnode buffer pool options */
-  struct {
-    /** write buffer size */
+  SDnode * pDnode;
   uint64_t wsize;
   uint64_t ssize;
   uint64_t lsize;
-    /** use heap allocator or arena allocator */
   bool     isHeapAllocator;
-  };
-
-  /** time to live of tables in this vnode */
   uint32_t ttl;
-
-  /** data to keep in this vnode */
   uint32_t keep;
-
-  /** if TS data is eventually consistency */
   bool     isWeak;
-
-  /** TSDB config */
   STsdbCfg tsdbCfg;
-
-  /** META config */
   SMetaCfg metaCfg;
-
-  /** TQ config */
   STqCfg   tqCfg;
-
-  /** WAL config */
   SWalCfg  walCfg;
 } SVnodeCfg;
 
 typedef struct {
   int32_t           sver;
-  char             *timezone;
-  char             *locale;
-  char             *charset;
+  char *            timezone;
+  char *            locale;
+  char *            charset;
   uint16_t          nthreads;  // number of commit threads. 0 for no threads and a schedule queue should be given (TODO)
   PutReqToVQueryQFp putReqToVQueryQFp;
 } SVnodeOpt;

source/dnode/vnode/meta/CMakeLists.txt

-set(META_DB_IMPL_LIST "BDB" "SQLITE")
-set(META_DB_IMPL "BDB" CACHE STRING "Use BDB as the default META implementation")
-set_property(CACHE META_DB_IMPL PROPERTY STRINGS ${META_DB_IMPL_LIST})
-
-if(META_DB_IMPL IN_LIST META_DB_IMPL_LIST)
-    message(STATUS "META DB Impl: ${META_DB_IMPL}==============")
-else()
-    message(FATAL_ERROR "Invalid META DB IMPL: ${META_DB_IMPL}==============")
-endif()
-
-aux_source_directory(src META_SRC)
-if(${META_DB_IMPL} STREQUAL "BDB")
-    list(REMOVE_ITEM META_SRC "src/metaSQLiteImpl.c")
-elseif(${META_DB_IMPL} STREQUAL "SQLITE")
-    list(REMOVE_ITEM META_SRC "src/metaBDBImpl.c")
-endif()
-
-add_library(meta STATIC ${META_SRC})
-target_include_directories(
-    meta
-    PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/vnode/meta"
-    PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/index"
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
-)
-target_link_libraries(
-    meta
-    PUBLIC common
-    PUBLIC index 
-)
-
-if(${META_DB_IMPL} STREQUAL "BDB")
-    target_link_libraries(
-        meta
-        PUBLIC bdb
-    )
-elseif(${META_DB_IMPL} STREQUAL "SQLITE")
-    target_link_libraries(
-        meta
-        PUBLIC sqlite
-    )
-endif()
-
-
-if(${BUILD_TEST})
-    add_subdirectory(test)
-endif(${BUILD_TEST})

source/dnode/vnode/meta/inc/metaCache.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_META_CACHE_H_
-#define _TD_META_CACHE_H_
-
-#include "meta.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct SMetaCache SMetaCache;
-
-int  metaOpenCache(SMeta *pMeta);
-void metaCloseCache(SMeta *pMeta);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_META_CACHE_H_*/
\ No newline at end of file

source/dnode/vnode/meta/inc/metaCfg.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_META_CFG_H_
-#define _TD_META_CFG_H_
-
-#include "meta.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-extern const SMetaCfg defaultMetaOptions;
-
-int  metaValidateOptions(const SMetaCfg *);
-void metaOptionsCopy(SMetaCfg *pDest, const SMetaCfg *pSrc);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_META_CFG_H_*/
\ No newline at end of file

source/dnode/vnode/meta/inc/metaDB.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_META_DB_H_
-#define _TD_META_DB_H_
-
-#include "meta.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct SMetaDB SMetaDB;
-
-int  metaOpenDB(SMeta *pMeta);
-void metaCloseDB(SMeta *pMeta);
-int  metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg);
-int  metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_META_DB_H_*/
\ No newline at end of file

source/dnode/vnode/meta/inc/metaIdx.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_META_IDX_H_
-#define _TD_META_IDX_H_
-
-#include "meta.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct SMetaIdx SMetaIdx;
-
-int  metaOpenIdx(SMeta *pMeta);
-void metaCloseIdx(SMeta *pMeta);
-int  metaSaveTableToIdx(SMeta *pMeta, const STbCfg *pTbOptions);
-int  metaRemoveTableFromIdx(SMeta *pMeta, tb_uid_t uid);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_META_IDX_H_*/
\ No newline at end of file

source/dnode/vnode/meta/inc/metaQuery.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _VNODE_QUERY_H_
-#define _VNODE_QUERY_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_VNODE_QUERY_H_*/
\ No newline at end of file

source/dnode/vnode/meta/inc/metaTbCfg.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_META_TABLE_CFG_H_
-#define _TD_META_TABLE_CFG_H_
-
-#include "meta.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-int    metaValidateTbCfg(SMeta *pMeta, const STbCfg *);
-size_t metaEncodeTbObjFromTbOptions(const STbCfg *, void *pBuf, size_t bsize);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_META_TABLE_CFG_H_*/
\ No newline at end of file

source/dnode/vnode/meta/inc/metaTbTag.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_META_TB_TAG_H_
-#define _TD_META_TB_TAG_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_META_TB_TAG_H_*/
\ No newline at end of file

source/dnode/vnode/meta/inc/metaTbUid.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_META_UID_H_
-#define _TD_META_UID_H_
-
-#include "meta.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* ------------------------ APIS EXPOSED ------------------------ */
-typedef struct STbUidGenerator {
-  tb_uid_t nextUid;
-} STbUidGenerator;
-
-// STableUidGenerator
-int  metaOpenUidGnrt(SMeta *pMeta);
-void metaCloseUidGnrt(SMeta *pMeta);
-
-// tb_uid_t
-#define IVLD_TB_UID 0
-tb_uid_t metaGenerateUid(SMeta *pMeta);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_META_UID_H_*/
\ No newline at end of file

source/dnode/vnode/meta/src/metaSQLiteImpl.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "metaDef.h"
-#include "sqlite3.h"
-
-struct SMetaDB {
-  sqlite3 *pDB;
-};
-
-int metaOpenDB(SMeta *pMeta) {
-  char  dir[128];
-  int   rc;
-  char *err = NULL;
-
-  pMeta->pDB = (SMetaDB *)calloc(1, sizeof(SMetaDB));
-  if (pMeta->pDB == NULL) {
-    // TODO: handle error
-    return -1;
-  }
-
-  sprintf(dir, "%s/meta.db", pMeta->path);
-  rc = sqlite3_open(dir, &(pMeta->pDB->pDB));
-  if (rc != SQLITE_OK) {
-    // TODO: handle error
-    printf("failed to open meta.db\n");
-  }
-
-  // For all tables
-  rc = sqlite3_exec(pMeta->pDB->pDB,
-                    "CREATE TABLE IF NOT EXISTS tb ("
-                    "  tbname VARCHAR(256) NOT NULL UNIQUE,"
-                    "  tb_uid INTEGER NOT NULL UNIQUE "
-                    ");",
-                    NULL, NULL, &err);
-  if (rc != SQLITE_OK) {
-    // TODO: handle error
-    printf("failed to create meta table tb since %s\n", err);
-  }
-
-  // For super tables
-  rc = sqlite3_exec(pMeta->pDB->pDB,
-                    "CREATE TABLE IF NOT EXISTS stb ("
-                    "    tb_uid INTEGER NOT NULL UNIQUE,"
-                    "    tbname VARCHAR(256) NOT NULL UNIQUE,"
-                    "    tb_schema BLOB NOT NULL,"
-                    "    tag_schema BLOB NOT NULL"
-                    ");",
-                    NULL, NULL, &err);
-  if (rc != SQLITE_OK) {
-    // TODO: handle error
-    printf("failed to create meta table stb since %s\n", err);
-  }
-
-  // For normal tables
-  rc = sqlite3_exec(pMeta->pDB->pDB,
-                    "CREATE TABLE IF NOT EXISTS ntb ("
-                    "    tb_uid INTEGER NOT NULL UNIQUE,"
-                    "    tbname VARCHAR(256) NOT NULL,"
-                    "    tb_schema BLOB NOT NULL"
-                    ");",
-                    NULL, NULL, &err);
-  if (rc != SQLITE_OK) {
-    // TODO: handle error
-    printf("failed to create meta table ntb since %s\n", err);
-  }
-
-  sqlite3_exec(pMeta->pDB->pDB, "BEGIN;", NULL, NULL, &err);
-
-  tfree(err);
-
-  return 0;
-}
-
-void metaCloseDB(SMeta *pMeta) {
-  if (pMeta->pDB) {
-    sqlite3_exec(pMeta->pDB->pDB, "COMMIT;", NULL, NULL, NULL);
-    sqlite3_close(pMeta->pDB->pDB);
-    free(pMeta->pDB);
-    pMeta->pDB = NULL;
-  }
-
-  // TODO
-}
-
-int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbCfg) {
-  char          sql[256];
-  char *        err = NULL;
-  int           rc;
-  tb_uid_t      uid;
-  sqlite3_stmt *stmt;
-  char          buf[256];
-  void *        pBuf;
-
-  switch (pTbCfg->type) {
-    case META_SUPER_TABLE:
-      uid = pTbCfg->stbCfg.suid;
-      sprintf(sql,
-              "INSERT INTO tb VALUES (\'%s\', %" PRIu64
-              ");"
-              "CREATE TABLE IF NOT EXISTS stb_%" PRIu64
-              " ("
-              "    tb_uid INTEGER NOT NULL UNIQUE,"
-              "    tbname VARCHAR(256),"
-              "    tag1 INTEGER);",
-              pTbCfg->name, uid, uid);
-      rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err);
-      if (rc != SQLITE_OK) {
-        printf("failed to create normal table since %s\n", err);
-      }
-
-      sprintf(sql, "INSERT INTO stb VALUES (%" PRIu64 ", %s, ?, ?)", uid, pTbCfg->name);
-      sqlite3_prepare_v2(pMeta->pDB->pDB, sql, -1, &stmt, NULL);
-
-      pBuf = buf;
-      tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pSchema);
-      sqlite3_bind_blob(stmt, 1, buf, POINTER_DISTANCE(pBuf, buf), NULL);
-      pBuf = buf;
-      tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pTagSchema);
-      sqlite3_bind_blob(stmt, 2, buf, POINTER_DISTANCE(pBuf, buf), NULL);
-
-      sqlite3_step(stmt);
-
-      sqlite3_finalize(stmt);
-
-#if 0
-      sprintf(sql,
-              "INSERT INTO tb VALUES (?, ?);"
-              // "INSERT INTO stb VALUES (?, ?, ?, ?);"
-              // "CREATE TABLE IF NOT EXISTS stb_%" PRIu64
-              // " ("
-              // "    tb_uid INTEGER NOT NULL UNIQUE,"
-              // "    tbname VARCHAR(256),"
-              // "    tag1 INTEGER);"
-              ,
-              uid);
-      rc = sqlite3_prepare_v2(pMeta->pDB->pDB, sql, -1, &stmt, NULL);
-      if (rc != SQLITE_OK) {
-        return -1;
-      }
-      sqlite3_bind_text(stmt, 1, pTbCfg->name, -1, SQLITE_TRANSIENT);
-      sqlite3_bind_int64(stmt, 2, uid);
-      sqlite3_step(stmt);
-      sqlite3_finalize(stmt);
-
-
-      // sqlite3_bind_int64(stmt, 3, uid);
-      // sqlite3_bind_text(stmt, 4, pTbCfg->name, -1, SQLITE_TRANSIENT);
-      // pBuf = buf;
-      // tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pSchema);
-      // sqlite3_bind_blob(stmt, 5, buf, POINTER_DISTANCE(pBuf, buf), NULL);
-      // pBuf = buf;
-      // tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pTagSchema);
-      // sqlite3_bind_blob(stmt, 6, buf, POINTER_DISTANCE(pBuf, buf), NULL);
-
-      rc = sqliteVjj3_step(stmt);
-      if (rc != SQLITE_OK) {
-        printf("failed to create normal table since %s\n", sqlite3_errmsg(pMeta->pDB->pDB));
-      }
-      sqlite3_finalize(stmt);
-#endif
-      break;
-    case META_NORMAL_TABLE:
-      // uid = metaGenerateUid(pMeta);
-      // sprintf(sql,
-      //         "INSERT INTO tb VALUES (\'%s\', %" PRIu64
-      //         ");"
-      //         "INSERT INTO ntb VALUES (%" PRIu64 ", \'%s\', );",
-      //         pTbCfg->name, uid, uid, pTbCfg->name, );
-
-      // rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err);
-      // if (rc != SQLITE_OK) {
-      //   printf("failed to create normal table since %s\n", err);
-      // }
-      break;
-    case META_CHILD_TABLE:
-#if 0
-      uid = metaGenerateUid(pMeta);
-      // sprintf(sql, "INSERT INTO tb VALUES (\'%s\', %" PRIu64
-      //              ");"
-      //              "INSERT INTO stb_%" PRIu64 " VALUES (%" PRIu64 ", \'%s\', );");
-      rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err);
-      if (rc != SQLITE_OK) {
-        printf("failed to create child table since %s\n", err);
-      }
-#endif
-      break;
-    default:
-      break;
-  }
-
-  tfree(err);
-
-  return 0;
-}
-
-int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) {
-  /* TODO */
-  return 0;
-}
\ No newline at end of file

source/dnode/vnode/meta/test/CMakeLists.txt

-# add_executable(metaTest "")
-# target_sources(metaTest
-#     PRIVATE
-#     "../src/metaMain.c"
-#     "../src/metaUid.c"
-#     "metaTests.cpp"
-# )
-# target_include_directories(metaTest 
-#     PUBLIC
-#     "${CMAKE_SOURCE_DIR}/include/server/vnode/meta"
-#     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
-# )
-# target_link_libraries(metaTest
-#     os
-#     util
-#     common
-#     gtest_main
-#     tkv
-# )
-# enable_testing()
-# add_test(
-#     NAME meta_test
-#     COMMAND metaTest
-# )

source/dnode/vnode/meta/test/metaAPITest.cpp

-#if 0
-#include <gtest/gtest.h>
-#include <string.h>
-#include <iostream>
-
-#include "meta.h"
-
-static STSchema *metaGetSimpleSchema() {
-  STSchema *      pSchema = NULL;
-  STSchemaBuilder sb = {0};
-
-  tdInitTSchemaBuilder(&sb, 0);
-  tdAddColToSchema(&sb, TSDB_DATA_TYPE_TIMESTAMP, 0, 8);
-  tdAddColToSchema(&sb, TSDB_DATA_TYPE_INT, 1, 4);
-
-  pSchema = tdGetSchemaFromBuilder(&sb);
-  tdDestroyTSchemaBuilder(&sb);
-
-  return pSchema;
-}
-
-static SKVRow metaGetSimpleTags() {
-  SKVRowBuilder kvrb = {0};
-  SKVRow        row;
-
-  tdInitKVRowBuilder(&kvrb);
-  int64_t ts = 1634287978000;
-  int32_t a = 10;
-
-  tdAddColToKVRow(&kvrb, 0, TSDB_DATA_TYPE_TIMESTAMP, (void *)(&ts));
-  tdAddColToKVRow(&kvrb, 0, TSDB_DATA_TYPE_INT, (void *)(&a));
-
-  row = tdGetKVRowFromBuilder(&kvrb);
-
-  tdDestroyKVRowBuilder(&kvrb);
-
-  return row;
-}
-
-TEST(MetaTest, DISABLED_meta_create_1m_normal_tables_test) {
-  // Open Meta
-  SMeta *meta = metaOpen(NULL, NULL);
-  std::cout << "Meta is opened!" << std::endl;
-
-  // Create 1000000 normal tables
-  META_TABLE_OPTS_DECLARE(tbOpts);
-  STSchema *pSchema = metaGetSimpleSchema();
-  char      tbname[128];
-
-  for (size_t i = 0; i < 1000000; i++) {
-    sprintf(tbname, "ntb%ld", i);
-    metaNormalTableOptsInit(&tbOpts, tbname, pSchema);
-    metaCreateTable(meta, &tbOpts);
-    metaTableOptsClear(&tbOpts);
-  }
-
-  tdFreeSchema(pSchema);
-
-  // Close Meta
-  metaClose(meta);
-  std::cout << "Meta is closed!" << std::endl;
-
-  // Destroy Meta
-  metaDestroy("meta");
-  std::cout << "Meta is destroyed!" << std::endl;
-}
-
-TEST(MetaTest, meta_create_1m_child_tables_test) {
-  // Open Meta
-  SMeta *meta = metaOpen(NULL);
-  std::cout << "Meta is opened!" << std::endl;
-
-  // Create a super tables
-  tb_uid_t uid = 477529885843758ul;
-  META_TABLE_OPTS_DECLARE(tbOpts);
-  STSchema *pSchema = metaGetSimpleSchema();
-  STSchema *pTagSchema = metaGetSimpleSchema();
-
-  metaSuperTableOptsInit(&tbOpts, "st", uid, pSchema, pTagSchema);
-  metaCreateTable(meta, &tbOpts);
-  metaTableOptsClear(&tbOpts);
-
-  tdFreeSchema(pSchema);
-  tdFreeSchema(pTagSchema);
-
-  // Create 1000000 child tables
-  char   name[128];
-  SKVRow row = metaGetSimpleTags();
-  for (size_t i = 0; i < 1000000; i++) {
-    sprintf(name, "ctb%ld", i);
-    metaChildTableOptsInit(&tbOpts, name, uid, row);
-    metaCreateTable(meta, &tbOpts);
-    metaTableOptsClear(&tbOpts);
-  }
-  kvRowFree(row);
-
-  // Close Meta
-  metaClose(meta);
-  std::cout << "Meta is closed!" << std::endl;
-
-  // Destroy Meta
-  metaDestroy("meta");
-  std::cout << "Meta is destroyed!" << std::endl;
-}
-#endif
\ No newline at end of file

source/dnode/vnode/meta/inc/metaDef.h → source/dnode/vnode/src/inc/metaDef.h

@@ -19,18 +19,49 @@
 #include "mallocator.h"
 
 #include "meta.h"
-#include "metaCache.h"
-#include "metaCfg.h"
-#include "metaDB.h"
-#include "metaIdx.h"
-#include "metaTbCfg.h"
-#include "metaTbTag.h"
-#include "metaTbUid.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+typedef struct SMetaCache SMetaCache;
+typedef struct SMetaIdx   SMetaIdx;
+typedef struct SMetaDB    SMetaDB;
+
+// SMetaDB
+int  metaOpenDB(SMeta* pMeta);
+void metaCloseDB(SMeta* pMeta);
+int  metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg);
+int  metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid);
+
+// SMetaCache
+int  metaOpenCache(SMeta* pMeta);
+void metaCloseCache(SMeta* pMeta);
+
+// SMetaCfg
+extern const SMetaCfg defaultMetaOptions;
+// int                   metaValidateOptions(const SMetaCfg*);
+void metaOptionsCopy(SMetaCfg* pDest, const SMetaCfg* pSrc);
+
+// SMetaIdx
+int  metaOpenIdx(SMeta* pMeta);
+void metaCloseIdx(SMeta* pMeta);
+int  metaSaveTableToIdx(SMeta* pMeta, const STbCfg* pTbOptions);
+int  metaRemoveTableFromIdx(SMeta* pMeta, tb_uid_t uid);
+
+// STbUidGnrt
+typedef struct STbUidGenerator {
+  tb_uid_t nextUid;
+} STbUidGenerator;
+
+// STableUidGenerator
+int  metaOpenUidGnrt(SMeta* pMeta);
+void metaCloseUidGnrt(SMeta* pMeta);
+
+// tb_uid_t
+#define IVLD_TB_UID 0
+tb_uid_t metaGenerateUid(SMeta* pMeta);
+
 struct SMeta {
   char*                 path;
   SMetaCfg              options;

source/dnode/vnode/src/inc/vnd.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_DEF_H_
+#define _TD_VNODE_DEF_H_
+
+#include "mallocator.h"
+// #include "sync.h"
+#include "tcoding.h"
+#include "tfs.h"
+#include "tlist.h"
+#include "tlockfree.h"
+#include "tmacro.h"
+#include "wal.h"
+
+#include "vnode.h"
+
+#include "vnodeQuery.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SVState   SVState;
+typedef struct SVBufPool SVBufPool;
+
+typedef struct SVnodeTask {
+  TD_DLIST_NODE(SVnodeTask);
+  void* arg;
+  int (*execute)(void*);
+} SVnodeTask;
+
+typedef struct SVnodeMgr {
+  td_mode_flag_t vnodeInitFlag;
+  // For commit
+  bool            stop;
+  uint16_t        nthreads;
+  pthread_t*      threads;
+  pthread_mutex_t mutex;
+  pthread_cond_t  hasTask;
+  TD_DLIST(SVnodeTask) queue;
+  // For vnode Mgmt
+  SDnode*           pDnode;
+  PutReqToVQueryQFp putReqToVQueryQFp;
+} SVnodeMgr;
+
+extern SVnodeMgr vnodeMgr;
+
+// SVState
+struct SVState {
+  int64_t processed;
+  int64_t committed;
+  int64_t applied;
+};
+
+struct SVnode {
+  int32_t    vgId;
+  char*      path;
+  SVnodeCfg  config;
+  SVState    state;
+  SVBufPool* pBufPool;
+  SMeta*     pMeta;
+  STsdb*     pTsdb;
+  STQ*       pTq;
+  SWal*      pWal;
+  tsem_t     canCommit;
+  SQHandle*  pQuery;
+  SDnode*    pDnode;
+};
+
+int vnodeScheduleTask(SVnodeTask* task);
+
+int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq);
+
+// For Log
+extern int32_t vDebugFlag;
+
+#define vFatal(...)                                 \
+  do {                                              \
+    if (vDebugFlag & DEBUG_FATAL) {                 \
+      taosPrintLog("VND FATAL ", 255, __VA_ARGS__); \
+    }                                               \
+  } while (0)
+#define vError(...)                                 \
+  do {                                              \
+    if (vDebugFlag & DEBUG_ERROR) {                 \
+      taosPrintLog("VND ERROR ", 255, __VA_ARGS__); \
+    }                                               \
+  } while (0)
+#define vWarn(...)                                 \
+  do {                                             \
+    if (vDebugFlag & DEBUG_WARN) {                 \
+      taosPrintLog("VND WARN ", 255, __VA_ARGS__); \
+    }                                              \
+  } while (0)
+#define vInfo(...)                            \
+  do {                                        \
+    if (vDebugFlag & DEBUG_INFO) {            \
+      taosPrintLog("VND ", 255, __VA_ARGS__); \
+    }                                         \
+  } while (0)
+#define vDebug(...)                                     \
+  do {                                                  \
+    if (vDebugFlag & DEBUG_DEBUG) {                     \
+      taosPrintLog("VND ", tsdbDebugFlag, __VA_ARGS__); \
+    }                                                   \
+  } while (0)
+#define vTrace(...)                                     \
+  do {                                                  \
+    if (vDebugFlag & DEBUG_TRACE) {                     \
+      taosPrintLog("VND ", tsdbDebugFlag, __VA_ARGS__); \
+    }                                                   \
+  } while (0)
+
+// vnodeCfg.h
+extern const SVnodeCfg defaultVnodeOptions;
+
+int  vnodeValidateOptions(const SVnodeCfg*);
+void vnodeOptionsCopy(SVnodeCfg* pDest, const SVnodeCfg* pSrc);
+
+// For commit
+#define vnodeShouldCommit vnodeBufPoolIsFull
+int vnodeSyncCommit(SVnode* pVnode);
+int vnodeAsyncCommit(SVnode* pVnode);
+
+// SVBufPool
+
+int   vnodeOpenBufPool(SVnode* pVnode);
+void  vnodeCloseBufPool(SVnode* pVnode);
+int   vnodeBufPoolSwitch(SVnode* pVnode);
+int   vnodeBufPoolRecycle(SVnode* pVnode);
+void* vnodeMalloc(SVnode* pVnode, uint64_t size);
+bool  vnodeBufPoolIsFull(SVnode* pVnode);
+
+SMemAllocatorFactory* vBufPoolGetMAF(SVnode* pVnode);
+
+// SVMemAllocator
+typedef struct SVArenaNode {
+  TD_SLIST_NODE(SVArenaNode);
+  uint64_t size;  // current node size
+  void*    ptr;
+  char     data[];
+} SVArenaNode;
+
+typedef struct SVMemAllocator {
+  T_REF_DECLARE()
+  TD_DLIST_NODE(SVMemAllocator);
+  uint64_t     capacity;
+  uint64_t     ssize;
+  uint64_t     lsize;
+  SVArenaNode* pNode;
+  TD_SLIST(SVArenaNode) nlist;
+} SVMemAllocator;
+
+SVMemAllocator* vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize);
+void            vmaDestroy(SVMemAllocator* pVMA);
+void            vmaReset(SVMemAllocator* pVMA);
+void*           vmaMalloc(SVMemAllocator* pVMA, uint64_t size);
+void            vmaFree(SVMemAllocator* pVMA, void* ptr);
+bool            vmaIsFull(SVMemAllocator* pVMA);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_DEF_H_*/
\ No newline at end of file

source/dnode/vnode/impl/inc/vnodeQuery.h → source/dnode/vnode/src/inc/vnodeQuery.h

@@ -19,11 +19,11 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-#include "vnodeInt.h"
 #include "qworker.h"
+#include "vnode.h"
 
-typedef struct SQWorkerMgmt SQHandle;
 
+typedef struct SQWorkerMgmt SQHandle;
 
 int vnodeQueryOpen(SVnode *pVnode);
 

source/dnode/vnode/meta/src/metaIdx.c → source/dnode/vnode/src/meta/metaIdx.c

@@ -13,7 +13,9 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#ifdef USE_INVERTED_INDEX
 #include "index.h"
+#endif
 #include "metaDef.h"
 
 struct SMetaIdx {

source/dnode/vnode/meta/src/metaMain.c → source/dnode/vnode/src/meta/metaMain.c

@@ -30,11 +30,11 @@ SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory
     pMetaCfg = &defaultMetaOptions;
   }
 
-  // Validate the options
-  if (metaValidateOptions(pMetaCfg) < 0) {
-    // TODO: deal with error
-    return NULL;
-  }
+  // // Validate the options
+  // if (metaValidateOptions(pMetaCfg) < 0) {
+  //   // TODO: deal with error
+  //   return NULL;
+  // }
 
   // Allocate handle
   pMeta = metaNew(path, pMetaCfg, pMAF);

source/dnode/vnode/meta/src/metaTable.c → source/dnode/vnode/src/meta/metaTable.c

@@ -17,10 +17,10 @@
 
 int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg) {
   // Validate the tbOptions
-  if (metaValidateTbCfg(pMeta, pTbCfg) < 0) {
-    // TODO: handle error
-    return -1;
-  }
+  // if (metaValidateTbCfg(pMeta, pTbCfg) < 0) {
+  //   // TODO: handle error
+  //   return -1;
+  // }
 
   // TODO: add atomicity