Merge branch '3.0' into feature/vnode

CMakeLists.txt

@@ -26,8 +26,9 @@ if(${BUILD_TEST})
 endif(${BUILD_TEST})
 add_subdirectory(source)
 add_subdirectory(tools)
+add_subdirectory(tests)
 
 # docs 
 add_subdirectory(docs)
 
-# tests (TODO)
+# tests (TODO)
\ No newline at end of file

cmake/cmake.options

@@ -37,6 +37,7 @@ option(
     off 
 )
 
+
 option(
     BUILD_WITH_NURAFT
     "If build with NuRaft" 
@@ -54,3 +55,9 @@ option(
     "If use doxygen build documents" 
     OFF
 )
+
+option(
+   BUILD_WITH_INVERTEDINDEX
+   "If use invertedIndex" 
+   ON 
+)

include/common/taosmsg.h

@@ -750,31 +750,36 @@ typedef struct {
 } SReplica;
 
 typedef struct {
-  char     db[TSDB_FULL_DB_NAME_LEN];
   int32_t  vgId;
+  int32_t  dnodeId;
+  char     db[TSDB_FULL_DB_NAME_LEN];
+  uint64_t dbUid;
   int32_t  cacheBlockSize;
   int32_t  totalBlocks;
   int32_t  daysPerFile;
   int32_t  daysToKeep0;
   int32_t  daysToKeep1;
   int32_t  daysToKeep2;
-  int32_t  minRowsPerFileBlock;
-  int32_t  maxRowsPerFileBlock;
+  int32_t  minRows;
+  int32_t  maxRows;
+  int32_t  commitTime;
   int32_t  fsyncPeriod;
-  int8_t   reserved[16];
+  int8_t   walLevel;
   int8_t   precision;
   int8_t   compression;
-  int8_t   cacheLastRow;
-  int8_t   update;
-  int8_t   walLevel;
   int8_t   quorum;
+  int8_t   update;
+  int8_t   cacheLastRow;
   int8_t   replica;
   int8_t   selfIndex;
   SReplica replicas[TSDB_MAX_REPLICA];
 } SCreateVnodeMsg, SAlterVnodeMsg;
 
 typedef struct {
-  int32_t vgId;
+  int32_t  vgId;
+  int32_t  dnodeId;
+  char     db[TSDB_FULL_DB_NAME_LEN];
+  uint64_t dbUid;
 } SDropVnodeMsg, SSyncVnodeMsg, SCompactVnodeMsg;
 
 typedef struct {

include/common/tname.h

@@ -16,7 +16,7 @@
 #ifndef TDENGINE_TNAME_H
 #define TDENGINE_TNAME_H
 
-#include "taosmsg.h"
+//#include "taosmsg.h"
 
 #define TSDB_DB_NAME_T     1
 #define TSDB_TABLE_NAME_T  2
@@ -27,7 +27,7 @@
 
 typedef struct SName {
   uint8_t type;  //db_name_t, table_name_t
-  char    acctId[TSDB_ACCT_ID_LEN];
+  int32_t acctId;
   char    dbname[TSDB_DB_NAME_LEN];
   char    tname[TSDB_TABLE_NAME_LEN];
 } SName;
@@ -38,7 +38,7 @@ int32_t tNameLen(const SName* name);
 
 SName* tNameDup(const SName* name);
 
-bool tIsValidName(const SName* name);
+bool tNameIsValid(const SName* name);
 
 const char* tNameGetTableName(const SName* name);
 
@@ -50,14 +50,10 @@ bool tNameIsEmpty(const SName* name);
 
 void tNameAssign(SName* dst, const SName* src);
 
-int32_t tNameFromString(SName* dst, const char* str, uint32_t type);
-
-int32_t tNameSetAcctId(SName* dst, const char* acct);
+int32_t tNameSetDbName(SName* dst, int32_t acctId, const char* dbName, size_t nameLen);
 
-SSchema* tGetTbnameColumnSchema();
+int32_t tNameFromString(SName* dst, const char* str, uint32_t type);
 
-#if 0
-int32_t tNameSetDbName(SName* dst, const char* acct, SToken* dbToken);
-#endif
+int32_t tNameSetAcctId(SName* dst, int32_t acctId);
 
 #endif  // TDENGINE_TNAME_H

include/libs/index/index.h

@@ -24,6 +24,7 @@ extern "C" {
 #endif
 
 typedef struct SIndex SIndex;
+typedef struct SIndexTerm SIndexTerm;
 typedef struct SIndexOpts SIndexOpts;
 typedef struct SIndexMultiTermQuery SIndexMultiTermQuery;
 typedef struct SArray               SIndexMultiTerm;
@@ -35,7 +36,7 @@ typedef enum  {
    ADD_INDEX,    // add index on specify column
    DROP_INDEX,   // drop existed index 
    DROP_SATBLE   // drop stable 
-} SIndexColumnType;
+} SIndexOperOnColumn;
 
 typedef enum  { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType;
 typedef enum  { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2,QUERY_REGEX = 3} EIndexQueryType;
@@ -45,12 +46,12 @@ typedef enum  { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2,QUERY_REGEX =
 */
 SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType oper);
 void            indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery);
-int             indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, const char *field, int32_t nFields, const char *value, int32_t nValue, EIndexQueryType type);
+int             indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType type);
 /* 
  * @param:    
  * @param:
  */
-SIndex* indexOpen(SIndexOpts *opt, const char *path);
+int   indexOpen(SIndexOpts *opt, const char *path, SIndex **index);
 void  indexClose(SIndex *index);
 int   indexPut(SIndex *index,    SIndexMultiTerm *terms, int uid);
 int   indexDelete(SIndex *index, SIndexMultiTermQuery *query); 
@@ -61,8 +62,8 @@ int   indexRebuild(SIndex *index, SIndexOpts *opt);
  * @param
  */
 SIndexMultiTerm *indexMultiTermCreate(); 
-int     indexMultiTermAdd(SIndexMultiTerm *terms, const char *field, int32_t nFields, const char *value, int32_t nValue);
-void    indexMultiTermDestroy(SIndexMultiTerm *terms);
+int indexMultiTermAdd(SIndexMultiTerm  *terms, SIndexTerm *term);
+void indexMultiTermDestroy(SIndexMultiTerm *terms);
 /*
  * @param: 
  * @param:
@@ -70,6 +71,17 @@ void    indexMultiTermDestroy(SIndexMultiTerm *terms);
 SIndexOpts *indexOptsCreate();
 void       indexOptsDestroy(SIndexOpts *opts);
 
+
+/*
+ * @param:
+ * @param:
+ */
+
+SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, 
+                            const char *colName, int32_t nColName, const char *colVal, int32_t nColVal);
+void        indexTermDestroy(SIndexTerm *p);
+
+
 #ifdef __cplusplus
 }
 #endif

include/libs/planner/planner.h

@@ -108,7 +108,7 @@ typedef struct SProjectPhyNode {
 typedef struct SExchangePhyNode {
   SPhyNode    node;
   uint64_t    srcTemplateId; // template id of datasource suplans
-  SArray     *pSourceEpSet;  // SEpSet, scheduler fill by calling qSetSuplanExecutionNode
+  SArray     *pSrcEndPoints;  // SEpAddrMsg, scheduler fill by calling qSetSuplanExecutionNode
 } SExchangePhyNode;
 
 typedef struct SSubplanId {
@@ -128,6 +128,8 @@ typedef struct SSubplan {
 } SSubplan;
 
 typedef struct SQueryDag {
+  uint64_t queryId;
+  int32_t  numOfSubplans;
   SArray  *pSubplans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0.
 } SQueryDag;
 
@@ -136,7 +138,11 @@ typedef struct SQueryDag {
  */
 int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag);
 
-int32_t qSetSuplanExecutionNode(SSubplan* subplan, SArray* nodes);
+// Set datasource of this subplan, multiple calls may be made to a subplan.
+// @subplan subplan to be schedule
+// @templateId templateId of a group of datasource subplans of this @subplan
+// @eps Execution location of this group of datasource subplans, is an array of SEpAddr structures 
+int32_t qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SArray* eps);
 
 int32_t qExplainQuery(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, char** str);
 
@@ -147,12 +153,14 @@ int32_t qSubPlanToString(const SSubplan* subplan, char** str);
 
 int32_t qStringToSubplan(const char* str, SSubplan** subplan);
 
+void qDestroySubplan(SSubplan* pSubplan);
+
 /**
  * Destroy the physical plan.
  * @param pQueryPhyNode
  * @return
  */
-void qDestroyQueryDag(struct SQueryDag* pDag);
+void qDestroyQueryDag(SQueryDag* pDag);
 
 #ifdef __cplusplus
 }

include/libs/query/query.h → include/libs/qcom/query.h

@@ -81,12 +81,14 @@ typedef struct STableMetaOutput {
   STableMeta *tbMeta;
 } STableMetaOutput;
 
+bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags);
+
 extern int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen);
 extern int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize);
 
+SSchema* tGetTbnameColumnSchema();
 extern void msgInit();
 
-
 extern int32_t qDebugFlag;
 
 #define qFatal(...)  do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0)

include/util/taoserror.h

@@ -120,17 +120,18 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TSC_INVALID_INPUT             TAOS_DEF_ERROR_CODE(0, 0X0224)  //"Invalid tsc input")
 
 // mnode-common
-#define TSDB_CODE_MND_NOT_READY                 TAOS_DEF_ERROR_CODE(0, 0x0300)
-#define TSDB_CODE_MND_MSG_NOT_PROCESSED         TAOS_DEF_ERROR_CODE(0, 0x0301)
-#define TSDB_CODE_MND_ACTION_IN_PROGRESS        TAOS_DEF_ERROR_CODE(0, 0x0302)
-#define TSDB_CODE_MND_ACTION_NEED_REPROCESSED   TAOS_DEF_ERROR_CODE(0, 0x0303)
-#define TSDB_CODE_MND_NO_RIGHTS                 TAOS_DEF_ERROR_CODE(0, 0x0304)
-#define TSDB_CODE_MND_INVALID_OPTIONS           TAOS_DEF_ERROR_CODE(0, 0x0305)
-#define TSDB_CODE_MND_INVALID_CONNECTION        TAOS_DEF_ERROR_CODE(0, 0x0306)
-#define TSDB_CODE_MND_INVALID_MSG_VERSION       TAOS_DEF_ERROR_CODE(0, 0x0307)
-#define TSDB_CODE_MND_INVALID_MSG_LEN           TAOS_DEF_ERROR_CODE(0, 0x0308)
-#define TSDB_CODE_MND_INVALID_MSG_TYPE          TAOS_DEF_ERROR_CODE(0, 0x0309)
-#define TSDB_CODE_MND_TOO_MANY_SHELL_CONNS      TAOS_DEF_ERROR_CODE(0, 0x030A)
+#define TSDB_CODE_MND_APP_ERROR                 TAOS_DEF_ERROR_CODE(0, 0x0300)
+#define TSDB_CODE_MND_NOT_READY                 TAOS_DEF_ERROR_CODE(0, 0x0301)
+#define TSDB_CODE_MND_MSG_NOT_PROCESSED         TAOS_DEF_ERROR_CODE(0, 0x0302)
+#define TSDB_CODE_MND_ACTION_IN_PROGRESS        TAOS_DEF_ERROR_CODE(0, 0x0303)
+#define TSDB_CODE_MND_ACTION_NEED_REPROCESSED   TAOS_DEF_ERROR_CODE(0, 0x0304)
+#define TSDB_CODE_MND_NO_RIGHTS                 TAOS_DEF_ERROR_CODE(0, 0x0305)
+#define TSDB_CODE_MND_INVALID_OPTIONS           TAOS_DEF_ERROR_CODE(0, 0x0306)
+#define TSDB_CODE_MND_INVALID_CONNECTION        TAOS_DEF_ERROR_CODE(0, 0x0307)
+#define TSDB_CODE_MND_INVALID_MSG_VERSION       TAOS_DEF_ERROR_CODE(0, 0x0308)
+#define TSDB_CODE_MND_INVALID_MSG_LEN           TAOS_DEF_ERROR_CODE(0, 0x0309)
+#define TSDB_CODE_MND_INVALID_MSG_TYPE          TAOS_DEF_ERROR_CODE(0, 0x030A)
+#define TSDB_CODE_MND_TOO_MANY_SHELL_CONNS      TAOS_DEF_ERROR_CODE(0, 0x030B)
 
 // mnode-show
 #define TSDB_CODE_MND_INVALID_SHOWOBJ           TAOS_DEF_ERROR_CODE(0, 0x0310)

include/util/tdef.h

@@ -231,10 +231,11 @@ do { \
 #define TSDB_DEFAULT_PAYLOAD_SIZE 5120   // default payload size, greater than PATH_MAX value
 #define TSDB_EXTRA_PAYLOAD_SIZE   128    // extra bytes for auth
 #define TSDB_CQ_SQL_SIZE          1024
-#define TSDB_MIN_VNODES           64
+#define TSDB_MIN_VNODES           16
 #define TSDB_MAX_VNODES           512
 #define TSDB_MIN_VNODES_PER_DB    1
 #define TSDB_MAX_VNODES_PER_DB    4096
+#define TSDB_DEFAULT_VN_PER_DB    2 
 
 #define TSDB_DNODE_ROLE_ANY       0
 #define TSDB_DNODE_ROLE_MGMT      1

source/client/CMakeLists.txt

@@ -2,14 +2,13 @@ aux_source_directory(src CLIENT_SRC)
 add_library(taos ${CLIENT_SRC})
 target_include_directories(
     taos
-        PUBLIC "${CMAKE_SOURCE_DIR}/include/client"
-        PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+    PUBLIC "${CMAKE_SOURCE_DIR}/include/client"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 target_link_libraries(
     taos
-    PRIVATE common
     INTERFACE api
-    PRIVATE os util common transport parser catalog function query
+    PRIVATE os util common transport parser catalog function qcom
 )
 
 ADD_SUBDIRECTORY(test)
\ No newline at end of file

source/client/inc/clientInt.h

@@ -76,11 +76,10 @@ typedef struct SAppInfo {
 typedef struct STscObj {
   char               user[TSDB_USER_LEN];
   char               pass[TSDB_PASSWORD_LEN];
-  char               acctId[TSDB_ACCT_ID_LEN];
   char               db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN];
+  int32_t            acctId;
   uint32_t           connId;
   uint64_t           id;       // ref ID returned by taosAddRef
-//  struct SSqlObj    *sqlList;
   void              *pTransporter;
   pthread_mutex_t    mutex;     // used to protect the operation on db
   int32_t            numOfReqs; // number of sqlObj from this tscObj

source/client/src/clientImpl.c

@@ -153,7 +153,7 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) {
     void*   output = NULL;
     int32_t outputLen = 0;
     code = qParseQuerySql(pRequest->sqlstr, sqlLen, pRequest->requestId, &type, &output, &outputLen, pRequest->msgBuf, ERROR_MSG_BUF_DEFAULT_SIZE);
-    if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW) {
+    if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW || type == TSDB_SQL_DROP_USER || type == TSDB_SQL_CREATE_DB) {
       pRequest->type = type;
       pRequest->body.param = output;
       pRequest->body.paramLen = outputLen;
@@ -430,8 +430,14 @@ void setResultDataPtr(SClientResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
   int32_t offset = 0;
   for (int32_t i = 0; i < numOfCols; ++i) {
     pResultInfo->length[i] = pResultInfo->fields[i].bytes;
-    pResultInfo->row[i]    = pResultInfo->pData + offset * pResultInfo->numOfRows;
+    pResultInfo->row[i]    = (char*) (pResultInfo->pData + offset * pResultInfo->numOfRows);
     pResultInfo->pCol[i]   = pResultInfo->row[i];
     offset += pResultInfo->fields[i].bytes;
   }
-}
+}
\ No newline at end of file
+
+const char *taos_get_client_info() { return version; }
+
+int taos_affected_rows(TAOS_RES *res) { return 1; }
+
+int taos_result_precision(TAOS_RES *res) { return TSDB_TIME_PRECISION_MILLI; }

source/client/src/clientMsgHandler.c

@@ -14,6 +14,7 @@
  */
 
 #include <catalog.h>
+#include <tname.h>
 #include "clientInt.h"
 #include "clientLog.h"
 #include "os.h"
@@ -2885,7 +2886,7 @@ int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVg
 }
 
 int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool autocreate, bool onlyLocal) {
-  assert(tIsValidName(&pTableMetaInfo->name));
+  assert(tNameIsValid(&pTableMetaInfo->name));
 
   char name[TSDB_TABLE_FNAME_LEN] = {0};
   tNameExtractFullName(&pTableMetaInfo->name, name);
@@ -3138,7 +3139,7 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) {
   // TODO refactor
   pthread_mutex_lock(&pTscObj->mutex);
   char temp[TSDB_TABLE_FNAME_LEN * 2] = {0};
-  int32_t len = sprintf(temp, "%s%s%s", pTscObj->acctId, TS_PATH_DELIMITER, pTscObj->db);
+  int32_t len = sprintf(temp, "%d%s%s", pTscObj->acctId, TS_PATH_DELIMITER, pTscObj->db);
 
   assert(len <= sizeof(pTscObj->db));
   tstrncpy(pTscObj->db, temp, sizeof(pTscObj->db));
@@ -3153,6 +3154,7 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) {
   }
 
   pTscObj->connId = pConnect->connId;
+  pTscObj->acctId = pConnect->acctId;
 
   // update the appInstInfo
   pTscObj->pAppInfo->clusterId = pConnect->clusterId;
@@ -3165,19 +3167,33 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) {
   return 0;
 }
 
-int32_t buildCreateUserMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) {
-  pMsgBody->msgType         = TSDB_MSG_TYPE_CREATE_USER;
-  pMsgBody->msgLen          = sizeof(SCreateUserMsg);
+int32_t doBuildMsgSupp(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) {
   pMsgBody->requestObjRefId = pRequest->self;
-  pMsgBody->pData           = pRequest->body.param;
-  return 0;
-}
-
-int32_t buildShowMsg(SRequestObj* pRequest, SRequestMsgBody* pMsgBody) {
-  pMsgBody->msgType         = TSDB_MSG_TYPE_SHOW;
   pMsgBody->msgLen          = pRequest->body.paramLen;
-  pMsgBody->requestObjRefId = pRequest->self;
   pMsgBody->pData           = pRequest->body.param;
+
+  switch(pRequest->type) {
+    case TSDB_SQL_CREATE_USER:
+      pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_USER;
+      break;
+    case TSDB_SQL_CREATE_DB: {
+      pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_DB;
+
+      SCreateDbMsg* pCreateMsg = pRequest->body.param;
+      SName name = {0};
+      int32_t ret = tNameSetDbName(&name, pRequest->pTscObj->acctId, pCreateMsg->db, strnlen(pCreateMsg->db, tListLen(pCreateMsg->db)));
+      if (ret != TSDB_CODE_SUCCESS) {
+        return -1;
+      }
+
+      tNameGetFullDbName(&name, pCreateMsg->db);
+
+      break;
+    }
+    case TSDB_SQL_SHOW:
+      pMsgBody->msgType = TSDB_MSG_TYPE_SHOW;
+      break;
+  }
 }
 
 STableMeta* createTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) {
@@ -3283,6 +3299,9 @@ int32_t processRetrieveMnodeRsp(SRequestObj *pRequest, const char* pMsg, int32_t
   return 0;
 }
 
+int32_t processCreateDbRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) {
+  // todo rsp with the vnode id list
+}
 
 void initMsgHandleFp() {
 #if 0
@@ -3363,11 +3382,14 @@ void initMsgHandleFp() {
   buildRequestMsgFp[TSDB_SQL_CONNECT]  = buildConnectMsg;
   handleRequestRspFp[TSDB_SQL_CONNECT] = processConnectRsp;
 
-  buildRequestMsgFp[TSDB_SQL_CREATE_USER]  = buildCreateUserMsg;
+  buildRequestMsgFp[TSDB_SQL_CREATE_USER]  = doBuildMsgSupp;
 
-  buildRequestMsgFp[TSDB_SQL_SHOW]         = buildShowMsg;
+  buildRequestMsgFp[TSDB_SQL_SHOW]         = doBuildMsgSupp;
   handleRequestRspFp[TSDB_SQL_SHOW]        = processShowRsp;
 
   buildRequestMsgFp[TSDB_SQL_RETRIEVE_MNODE] = buildRetrieveMnodeMsg;
   handleRequestRspFp[TSDB_SQL_RETRIEVE_MNODE]= processRetrieveMnodeRsp;
+
+  buildRequestMsgFp[TSDB_SQL_CREATE_DB]      = doBuildMsgSupp;
+  handleRequestRspFp[TSDB_SQL_CREATE_DB]     = processCreateDbRsp;
 }
\ No newline at end of file

source/client/test/clientTests.cpp

@@ -57,25 +57,38 @@ TEST(testCase, create_user_Test) {
   taos_close(pConn);
 }
 
-//TEST(testCase, show_user_Test) {
+//TEST(testCase, drop_user_Test) {
 //  TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
 //  assert(pConn != NULL);
 //
-//  TAOS_RES* pRes = taos_query(pConn, "show users");
-//  TAOS_ROW pRow = NULL;
-//
-//  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
-//  int32_t numOfFields = taos_num_fields(pRes);
-//
-//  char str[512] = {0};
-//  while((pRow = taos_fetch_row(pRes)) != NULL) {
-//    int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
-//    printf("%s\n", str);
+//  TAOS_RES* pRes = taos_query(pConn, "drop user abc");
+//  if (taos_errno(pRes) != TSDB_CODE_SUCCESS) {
+//    printf("failed to create user, reason:%s\n", taos_errstr(pRes));
 //  }
 //
+//  taos_free_result(pRes);
 //  taos_close(pConn);
 //}
 
+TEST(testCase, show_user_Test) {
+  TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
+  assert(pConn != NULL);
+
+  TAOS_RES* pRes = taos_query(pConn, "show users");
+  TAOS_ROW pRow = NULL;
+
+  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
+  int32_t numOfFields = taos_num_fields(pRes);
+
+  char str[512] = {0};
+  while((pRow = taos_fetch_row(pRes)) != NULL) {
+    int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
+    printf("%s\n", str);
+  }
+
+  taos_close(pConn);
+}
+
 TEST(testCase, show_db_Test) {
   TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
   assert(pConn != NULL);
@@ -94,3 +107,18 @@ TEST(testCase, show_db_Test) {
 
   taos_close(pConn);
 }
+
+TEST(testCase, create_db_Test) {
+  TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
+  assert(pConn != NULL);
+
+  TAOS_RES* pRes = taos_query(pConn, "create database abc");
+
+  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
+  ASSERT_TRUE(pFields == NULL);
+
+  int32_t numOfFields = taos_num_fields(pRes);
+  ASSERT_EQ(numOfFields, 0);
+
+  taos_close(pConn);
+}

source/common/inc/commonInt.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_COMMON_INT_H_
-#define _TD_COMMON_INT_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-extern bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags);
-
-
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_COMMON_INT_H_*/

source/common/src/tname.c

@@ -4,9 +4,6 @@
 #include "tname.h"
 #include "taosmsg.h"
 
-#define VALIDNUMOFCOLS(x)  ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS)
-#define VALIDNUMOFTAGS(x)  ((x) >= 0 && (x) <= TSDB_MAX_TAGS)
-
 #define VALID_NAME_TYPE(x)  ((x) == TSDB_DB_NAME_T || (x) == TSDB_TABLE_NAME_T)
 
 char* extractDBName(const char* tableId, char* name) {
@@ -120,84 +117,11 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in
 
 #endif
 
-static struct SSchema _s = {
-    .colId = TSDB_TBNAME_COLUMN_INDEX,
-    .type  = TSDB_DATA_TYPE_BINARY,
-    .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE,
-    .name = "tbname",
-};
-
-SSchema* tGetTbnameColumnSchema() {
-  return &_s;
-}
-
-static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen) {
-  int32_t rowLen = 0;
-
-  for (int32_t i = 0; i < numOfCols; ++i) {
-    // 1. valid types
-    if (!isValidDataType(pSchema[i].type)) {
-      return false;
-    }
-
-    // 2. valid length for each type
-    if (pSchema[i].type == TSDB_DATA_TYPE_BINARY) {
-      if (pSchema[i].bytes > TSDB_MAX_BINARY_LEN) {
-        return false;
-      }
-    } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
-      if (pSchema[i].bytes > TSDB_MAX_NCHAR_LEN) {
-        return false;
-      }
-    } else {
-      if (pSchema[i].bytes != tDataTypes[pSchema[i].type].bytes) {
-        return false;
-      }
-    }
-
-    // 3. valid column names
-    for (int32_t j = i + 1; j < numOfCols; ++j) {
-      if (strncasecmp(pSchema[i].name, pSchema[j].name, sizeof(pSchema[i].name) - 1) == 0) {
-        return false;
-      }
-    }
-
-    rowLen += pSchema[i].bytes;
-  }
-
-  return rowLen <= maxLen;
-}
-
-bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags) {
-  if (!VALIDNUMOFCOLS(numOfCols)) {
-    return false;
-  }
-
-  if (!VALIDNUMOFTAGS(numOfTags)) {
-    return false;
-  }
-
-  /* first column must be the timestamp, which is a primary key */
-  if (pSchema[0].type != TSDB_DATA_TYPE_TIMESTAMP) {
-    return false;
-  }
-
-  if (!doValidateSchema(pSchema, numOfCols, TSDB_MAX_BYTES_PER_ROW)) {
-    return false;
-  }
-
-  if (!doValidateSchema(&pSchema[numOfCols], numOfTags, TSDB_MAX_TAGS_LEN)) {
-    return false;
-  }
-
-  return true;
-}
-
 int32_t tNameExtractFullName(const SName* name, char* dst) {
   assert(name != NULL && dst != NULL);
 
   // invalid full name format, abort
-  if (!tIsValidName(name)) {
+  if (!tNameIsValid(name)) {
     return -1;
   }
 
@@ -230,7 +154,7 @@ int32_t tNameLen(const SName* name) {
   }
 }
 
-bool tIsValidName(const SName* name) {
+bool tNameIsValid(const SName* name) {
   assert(name != NULL);
 
   if (!VALID_NAME_TYPE(name->type)) {
@@ -265,13 +189,13 @@ int32_t tNameGetDbName(const SName* name, char* dst) {
 int32_t tNameGetFullDbName(const SName* name, char* dst) {
   assert(name != NULL && dst != NULL);
   snprintf(dst, TSDB_ACCT_ID_LEN + TS_PATH_DELIMITER_LEN + TSDB_DB_NAME_LEN,    // there is a over write risk
-      "%s.%s", name->acctId, name->dbname);
+      "%d.%s", name->acctId, name->dbname);
   return 0;
 }
 
 bool tNameIsEmpty(const SName* name) {
   assert(name != NULL);
-  return name->type == 0 || strlen(name->acctId) <= 0;
+  return name->type == 0 || name->acctId == 0;
 }
 
 const char* tNameGetTableName(const SName* name) {
@@ -283,32 +207,23 @@ void tNameAssign(SName* dst, const SName* src) {
   memcpy(dst, src, sizeof(SName));
 }
 
-//int32_t tNameSetDbName(SName* dst, const char* acct, SStrToken* dbToken) {
-//  assert(dst != NULL && dbToken != NULL && acct != NULL);
-//
-//  // too long account id or too long db name
-//  if (strlen(acct) >= tListLen(dst->acctId) || dbToken->n >= tListLen(dst->dbname)) {
-//    return -1;
-//  }
-//
-//  dst->type = TSDB_DB_NAME_T;
-//  tstrncpy(dst->acctId, acct, tListLen(dst->acctId));
-//  tstrncpy(dst->dbname, dbToken->z, dbToken->n + 1);
-//  return 0;
-//}
-
-int32_t tNameSetAcctId(SName* dst, const char* acct) {
-  assert(dst != NULL && acct != NULL);
+int32_t tNameSetDbName(SName* dst, int32_t acct, const char* dbName, size_t nameLen) {
+  assert(dst != NULL && dbName != NULL && nameLen > 0);
 
   // too long account id or too long db name
-  if (strlen(acct) >= tListLen(dst->acctId)) {
+  if (nameLen >= tListLen(dst->dbname)) {
     return -1;
   }
 
-  tstrncpy(dst->acctId, acct, tListLen(dst->acctId));
+  dst->type = TSDB_DB_NAME_T;
+  dst->acctId = acct;
+  tstrncpy(dst->dbname, dbName, nameLen + 1);
+  return 0;
+}
 
-  assert(strlen(dst->acctId) > 0);
-  
+int32_t tNameSetAcctId(SName* dst, int32_t acctId) {
+  assert(dst != NULL && acct != NULL);
+  dst->acctId = acctId;
   return 0;
 }
 
@@ -325,14 +240,11 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) {
     int32_t len = (int32_t)(p - str);
 
     // too long account id or too long db name
-    if ((len >= tListLen(dst->acctId)) || (len <= 0)) {
-      return -1;
-    }
-
-    memcpy (dst->acctId, str, len);
-    dst->acctId[len] = 0;
-
-    assert(strlen(dst->acctId) > 0);
+//    if ((len >= tListLen(dst->acctId)) || (len <= 0)) {
+//      return -1;
+//    }
+//    memcpy (dst->acctId, str, len);
+    dst->acctId = strtoll(str, NULL, 10);
   }
 
   if ((type & T_NAME_DB) == T_NAME_DB) {

source/dnode/mgmt/daemon/src/daemon.c

@@ -92,6 +92,7 @@ void dmnPrintVersion() {
 }
 
 int dmnReadConfig(const char *path) {
+  tstrncpy(configDir, global.configDir, PATH_MAX);
   taosInitGlobalCfg();
   taosReadGlobalLogCfg();
 

source/dnode/mgmt/impl/src/dndVnodes.c

@@ -95,7 +95,7 @@ static int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg);
 
 static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId) {
   SVnodesMgmt *pMgmt = &pDnode->vmgmt;
-  SVnodeObj *  pVnode = NULL;
+  SVnodeObj   *pVnode = NULL;
   int32_t      refCount = 0;
 
   taosRLockLatch(&pMgmt->latch);
@@ -107,23 +107,23 @@ static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId) {
   }
   taosRUnLockLatch(&pMgmt->latch);
 
-  dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount);
+  if (pVnode != NULL) {
+    dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount);
+  }
+
   return pVnode;
 }
 
 static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode) {
+  if (pVnode == NULL) return;
+
   SVnodesMgmt *pMgmt = &pDnode->vmgmt;
-  int32_t      refCount = 0;
 
   taosRLockLatch(&pMgmt->latch);
-  if (pVnode != NULL) {
-    refCount = atomic_sub_fetch_32(&pVnode->refCount, 1);
-  }
+  int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1);
   taosRUnLockLatch(&pMgmt->latch);
 
-  if (pVnode != NULL) {
-    dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount);
-  }
+  dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount);
 }
 
 static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, SVnode *pImpl) {
@@ -457,7 +457,7 @@ static int32_t dndOpenVnodes(SDnode *pDnode) {
 
   pMgmt->totalVnodes = numOfVnodes;
 
-  int32_t threadNum = tsNumOfCores;
+  int32_t threadNum = pDnode->opt.numOfCores;
   int32_t vnodesPerThread = numOfVnodes / threadNum + 1;
 
   SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread));
@@ -525,33 +525,49 @@ static void dndCloseVnodes(SDnode *pDnode) {
 
 static int32_t dndParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg *pCfg) {
   SCreateVnodeMsg *pCreate = rpcMsg->pCont;
-  *vgId = htonl(pCreate->vgId);
+  pCreate->vgId = htonl(pCreate->vgId);
+  pCreate->dnodeId = htonl(pCreate->dnodeId);
+  pCreate->dbUid = htobe64(pCreate->dbUid);
+  pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize);
+  pCreate->totalBlocks = htonl(pCreate->totalBlocks);
+  pCreate->daysPerFile = htonl(pCreate->daysPerFile);
+  pCreate->daysToKeep0 = htonl(pCreate->daysToKeep0);
+  pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1);
+  pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2);
+  pCreate->minRows = htonl(pCreate->minRows);
+  pCreate->maxRows = htonl(pCreate->maxRows);
+  pCreate->commitTime = htonl(pCreate->commitTime);
+  pCreate->fsyncPeriod = htonl(pCreate->fsyncPeriod);
+  for (int r = 0; r < pCreate->replica; ++r) {
+    SReplica *pReplica = &pCreate->replicas[r];
+    pReplica->id = htonl(pReplica->id);
+    pReplica->port = htons(pReplica->port);
+  }
+
+  *vgId = pCreate->vgId;
 
 #if 0
-  tstrncpy(pCfg->db, pCreate->db, TSDB_FULL_DB_NAME_LEN);
-  pCfg->cacheBlockSize = htonl(pCreate->cacheBlockSize);
-  pCfg->totalBlocks = htonl(pCreate->totalBlocks);
-  pCfg->daysPerFile = htonl(pCreate->daysPerFile);
-  pCfg->daysToKeep0 = htonl(pCreate->daysToKeep0);
-  pCfg->daysToKeep1 = htonl(pCreate->daysToKeep1);
-  pCfg->daysToKeep2 = htonl(pCreate->daysToKeep2);
-  pCfg->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock);
-  pCfg->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock);
-  pCfg->precision = pCreate->precision;
-  pCfg->compression = pCreate->compression;
-  pCfg->cacheLastRow = pCreate->cacheLastRow;
-  pCfg->update = pCreate->update;
-  pCfg->quorum = pCreate->quorum;
-  pCfg->replica = pCreate->replica;
-  pCfg->walLevel = pCreate->walLevel;
-  pCfg->fsyncPeriod = htonl(pCreate->fsyncPeriod);
-
-  for (int32_t i = 0; i < pCfg->replica; ++i) {
-    pCfg->replicas[i].port = htons(pCreate->replicas[i].port);
-    tstrncpy(pCfg->replicas[i].fqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN);
-  }
+  pCfg->wsize = pCreate->cacheBlockSize;
+  pCfg->ssize = pCreate->cacheBlockSize;
+  pCfg->wsize = pCreate->cacheBlockSize;
+  pCfg->lsize = pCreate->cacheBlockSize;
+  pCfg->isHeapAllocator = true;
+  pCfg->ttl = 4;
+  pCfg->keep = pCreate->daysToKeep0;
+  pCfg->isWeak = true;
+  pCfg->tsdbCfg.keep0 = pCreate->daysToKeep0;
+  pCfg->tsdbCfg.keep1 = pCreate->daysToKeep2;
+  pCfg->tsdbCfg.keep2 = pCreate->daysToKeep0;
+  pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize;
+  pCfg->metaCfg.lruSize = pCreate->cacheBlockSize;
+  pCfg->walCfg.fsyncPeriod = pCreate->fsyncPeriod;
+  pCfg->walCfg.level = pCreate->walLevel;
+  pCfg->walCfg.retentionPeriod = 10;
+  pCfg->walCfg.retentionSize = 128;
+  pCfg->walCfg.rollPeriod = 128;
+  pCfg->walCfg.segSize = 128;
+  pCfg->walCfg.vgId = pCreate->vgId;
 #endif
-
   return 0;
 }
 
@@ -1016,7 +1032,7 @@ static int32_t dndInitVnodeWriteWorker(SDnode *pDnode) {
   SVnodesMgmt * pMgmt = &pDnode->vmgmt;
   SMWorkerPool *pPool = &pMgmt->writePool;
   pPool->name = "vnode-write";
-  pPool->max = tsNumOfCores;
+  pPool->max = pDnode->opt.numOfCores;
   if (tMWorkerInit(pPool) != 0) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
@@ -1050,7 +1066,7 @@ static void dndFreeVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode) {
 }
 
 static int32_t dndInitVnodeSyncWorker(SDnode *pDnode) {
-  int32_t maxThreads = tsNumOfCores / 2;
+  int32_t maxThreads = pDnode->opt.numOfCores / 2;
   if (maxThreads < 1) maxThreads = 1;
 
   SVnodesMgmt  *pMgmt = &pDnode->vmgmt;

source/dnode/mgmt/impl/test/CMakeLists.txt

@@ -15,6 +15,6 @@ add_subdirectory(stb)
 # add_subdirectory(telem)
 # add_subdirectory(trans)
 add_subdirectory(user)
-# add_subdirectory(vgroup)
+add_subdirectory(vgroup)
 
 # add_subdirectory(common)

source/dnode/mgmt/impl/test/db/db.cpp

@@ -232,6 +232,7 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) {
     SRpcMsg* pMsg = pClient->pRsp;
     ASSERT_NE(pMsg, nullptr);
     ASSERT_EQ(pMsg->code, 0);
+    // taosMsleep(1000000);
   }
 
   SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL);

source/dnode/mgmt/impl/test/vgroup/CMakeLists.txt

+add_executable(dnode_test_vgroup "")
+
+target_sources(dnode_test_vgroup
+    PRIVATE
+    "vgroup.cpp"
+    "../sut/deploy.cpp"
+)
+
+target_link_libraries(
+    dnode_test_vgroup
+    PUBLIC dnode
+    PUBLIC util
+    PUBLIC os
+    PUBLIC gtest_main
+)
+
+target_include_directories(dnode_test_vgroup
+    PUBLIC
+    "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../../inc"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../sut"
+)
+
+add_test(
+    NAME dnode_test_vgroup
+    COMMAND dnode_test_vgroup
+)

source/dnode/mgmt/impl/test/vgroup/vgroup.cpp

+/**
+ * @file db.cpp
+ * @author slguan (slguan@taosdata.com)
+ * @brief DNODE module vgroup-msg tests
+ * @version 0.1
+ * @date 2021-12-20
+ *
+ * @copyright Copyright (c) 2021
+ *
+ */
+
+#include "deploy.h"
+
+class DndTestVgroup : public ::testing::Test {
+ protected:
+  static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) {
+    SServer* pServer = createServer(path, fqdn, port, firstEp);
+    ASSERT(pServer);
+    return pServer;
+  }
+
+  static void SetUpTestSuite() {
+    initLog("/tmp/tdlog");
+
+    const char* fqdn = "localhost";
+    const char* firstEp = "localhost:9150";
+    pServer = CreateServer("/tmp/dnode_test_vgroup", fqdn, 9150, firstEp);
+    pClient = createClient("root", "taosdata", fqdn, 9150);
+    taosMsleep(1100);
+  }
+
+  static void TearDownTestSuite() {
+    stopServer(pServer);
+    dropClient(pClient);
+    pServer = NULL;
+    pClient = NULL;
+  }
+
+  static SServer* pServer;
+  static SClient* pClient;
+  static int32_t  connId;
+
+ public:
+  void SetUp() override {}
+  void TearDown() override {}
+
+  void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns, const char* db) {
+    SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg));
+    pShow->type = showType;
+    if (db != NULL) {
+      strcpy(pShow->db, db);
+    }
+    SRpcMsg showRpcMsg = {0};
+    showRpcMsg.pCont = pShow;
+    showRpcMsg.contLen = sizeof(SShowMsg);
+    showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW;
+
+    sendMsg(pClient, &showRpcMsg);
+    ASSERT_NE(pClient->pRsp, nullptr);
+    ASSERT_EQ(pClient->pRsp->code, 0);
+    ASSERT_NE(pClient->pRsp->pCont, nullptr);
+
+    SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont;
+    ASSERT_NE(pShowRsp, nullptr);
+    pShowRsp->showId = htonl(pShowRsp->showId);
+    pMeta = &pShowRsp->tableMeta;
+    pMeta->numOfTags = htonl(pMeta->numOfTags);
+    pMeta->numOfColumns = htonl(pMeta->numOfColumns);
+    pMeta->sversion = htonl(pMeta->sversion);
+    pMeta->tversion = htonl(pMeta->tversion);
+    pMeta->tuid = htobe64(pMeta->tuid);
+    pMeta->suid = htobe64(pMeta->suid);
+
+    showId = pShowRsp->showId;
+
+    EXPECT_NE(pShowRsp->showId, 0);
+    EXPECT_STREQ(pMeta->tbFname, showName);
+    EXPECT_EQ(pMeta->numOfTags, 0);
+    EXPECT_EQ(pMeta->numOfColumns, columns);
+    EXPECT_EQ(pMeta->precision, 0);
+    EXPECT_EQ(pMeta->tableType, 0);
+    EXPECT_EQ(pMeta->update, 0);
+    EXPECT_EQ(pMeta->sversion, 0);
+    EXPECT_EQ(pMeta->tversion, 0);
+    EXPECT_EQ(pMeta->tuid, 0);
+    EXPECT_EQ(pMeta->suid, 0);
+  }
+
+  void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) {
+    SSchema* pSchema = &pMeta->pSchema[index];
+    pSchema->bytes = htonl(pSchema->bytes);
+    EXPECT_EQ(pSchema->colId, 0);
+    EXPECT_EQ(pSchema->type, type);
+    EXPECT_EQ(pSchema->bytes, bytes);
+    EXPECT_STREQ(pSchema->name, name);
+  }
+
+  void SendThenCheckShowRetrieveMsg(int32_t rows) {
+    SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg));
+    pRetrieve->showId = htonl(showId);
+    pRetrieve->free = 0;
+
+    SRpcMsg retrieveRpcMsg = {0};
+    retrieveRpcMsg.pCont = pRetrieve;
+    retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg);
+    retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE;
+
+    sendMsg(pClient, &retrieveRpcMsg);
+
+    ASSERT_NE(pClient->pRsp, nullptr);
+    ASSERT_EQ(pClient->pRsp->code, 0);
+    ASSERT_NE(pClient->pRsp->pCont, nullptr);
+
+    pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont;
+    ASSERT_NE(pRetrieveRsp, nullptr);
+    pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows);
+    pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds);
+    pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen);
+
+    EXPECT_EQ(pRetrieveRsp->numOfRows, rows);
+    EXPECT_EQ(pRetrieveRsp->useconds, 0);
+    // EXPECT_EQ(pRetrieveRsp->completed, completed);
+    EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI);
+    EXPECT_EQ(pRetrieveRsp->compressed, 0);
+    EXPECT_EQ(pRetrieveRsp->compLen, 0);
+
+    pData = pRetrieveRsp->data;
+    pos = 0;
+  }
+
+  void CheckInt8(int8_t val) {
+    int8_t data = *((int8_t*)(pData + pos));
+    pos += sizeof(int8_t);
+    EXPECT_EQ(data, val);
+  }
+
+  void CheckInt16(int16_t val) {
+    int16_t data = *((int16_t*)(pData + pos));
+    pos += sizeof(int16_t);
+    EXPECT_EQ(data, val);
+  }
+
+  void CheckInt32(int32_t val) {
+    int32_t data = *((int32_t*)(pData + pos));
+    pos += sizeof(int32_t);
+    EXPECT_EQ(data, val);
+  }
+
+  void CheckInt64(int64_t val) {
+    int64_t data = *((int64_t*)(pData + pos));
+    pos += sizeof(int64_t);
+    EXPECT_EQ(data, val);
+  }
+
+  void CheckTimestamp() {
+    int64_t data = *((int64_t*)(pData + pos));
+    pos += sizeof(int64_t);
+    EXPECT_GT(data, 0);
+  }
+
+  void CheckBinary(const char* val, int32_t len) {
+    pos += sizeof(VarDataLenT);
+    char* data = (char*)(pData + pos);
+    pos += len;
+    EXPECT_STREQ(data, val);
+  }
+
+  int32_t            showId;
+  STableMetaMsg*     pMeta;
+  SRetrieveTableRsp* pRetrieveRsp;
+  char*              pData;
+  int32_t            pos;
+};
+
+SServer* DndTestVgroup::pServer;
+SClient* DndTestVgroup::pClient;
+int32_t  DndTestVgroup::connId;
+
+
+TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) {
+  {
+    SCreateVnodeMsg* pReq = (SCreateVnodeMsg*)rpcMallocCont(sizeof(SCreateVnodeMsg));
+    pReq->vgId = htonl(2);
+    pReq->dnodeId = htonl(1);
+    strcpy(pReq->db, "1.d1");
+    pReq->dbUid = htobe64(9527);
+    pReq->cacheBlockSize = htonl(16);
+    pReq->totalBlocks = htonl(10);
+    pReq->daysPerFile = htonl(10);
+    pReq->daysToKeep0 = htonl(3650);
+    pReq->daysToKeep1 = htonl(3650);
+    pReq->daysToKeep2 = htonl(3650);
+    pReq->minRows = htonl(100);
+    pReq->minRows = htonl(4096);
+    pReq->commitTime = htonl(3600);
+    pReq->fsyncPeriod = htonl(3000);
+    pReq->walLevel = 1;
+    pReq->precision = 0;
+    pReq->compression = 2;
+    pReq->replica = 1;
+    pReq->quorum = 1;
+    pReq->update = 0;
+    pReq->cacheLastRow = 0;
+    pReq->selfIndex = 0;
+    for (int r = 0; r < pReq->replica; ++r) {
+      SReplica* pReplica = &pReq->replicas[r];
+      pReplica->id = htonl(1);
+      pReplica->port = htons(9150);
+    }
+
+    SRpcMsg rpcMsg = {0};
+    rpcMsg.pCont = pReq;
+    rpcMsg.contLen = sizeof(SCreateVnodeMsg);
+    rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_VNODE_IN;
+
+    sendMsg(pClient, &rpcMsg);
+    SRpcMsg* pMsg = pClient->pRsp;
+    ASSERT_NE(pMsg, nullptr);
+    ASSERT_EQ(pMsg->code, 0);
+    taosMsleep(1000000);
+  }
+
+}
+

source/dnode/mnode/impl/inc/mndDef.h

@@ -182,6 +182,7 @@ typedef struct {
 } SUserObj;
 
 typedef struct {
+  int32_t numOfVgroups;
   int32_t cacheBlockSize;
   int32_t totalBlocks;
   int32_t daysPerFile;
@@ -209,7 +210,6 @@ typedef struct {
   int64_t uid;
   int32_t cfgVersion;
   int32_t vgVersion;
-  int32_t numOfVgroups;
   int8_t  hashMethod;  // default is 1
   SDbCfg  cfg;
 } SDbObj;

source/dnode/mnode/impl/inc/mndTrans.h

@@ -30,8 +30,8 @@ void    mndTransDrop(STrans *pTrans);
 int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw);
 int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw);
 int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw);
-int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *, void *pMsg);
-int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *, void *pMsg);
+int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont);
+int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont);
 int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans);
 void    mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code);
 char   *mndTransStageStr(ETrnStage stage);

source/dnode/mnode/impl/inc/mndVgroup.h

@@ -28,7 +28,9 @@ SVgObj  *mndAcquireVgroup(SMnode *pMnode, int32_t vgId);
 void     mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup);
 int32_t  mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups);
 SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup);
-SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw);
+
+SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup);
+SDropVnodeMsg   *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/src/mndDb.c

@@ -77,8 +77,8 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
   SDB_SET_INT64(pRaw, dataPos, pDb->uid)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfgVersion)
   SDB_SET_INT32(pRaw, dataPos, pDb->vgVersion)
-  SDB_SET_INT32(pRaw, dataPos, pDb->numOfVgroups)
   SDB_SET_INT8(pRaw, dataPos, pDb->hashMethod)
+  SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfVgroups)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.cacheBlockSize)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.totalBlocks)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysPerFile)
@@ -124,8 +124,8 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
   SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->uid)
   SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfgVersion)
   SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->vgVersion)
-  SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->numOfVgroups)
   SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->hashMethod)
+  SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.numOfVgroups)
   SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.cacheBlockSize)
   SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.totalBlocks)
   SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysPerFile)
@@ -163,7 +163,6 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) {
   pOldDb->updateTime = pNewDb->createdTime;
   pOldDb->cfgVersion = pNewDb->cfgVersion;
   pOldDb->vgVersion = pNewDb->vgVersion;
-  pOldDb->numOfVgroups = pNewDb->numOfVgroups;
   memcpy(&pOldDb->cfg, &pNewDb->cfg, sizeof(SDbCfg));
   return 0;
 }
@@ -195,6 +194,7 @@ static int32_t mndCheckDbName(char *dbName, SUserObj *pUser) {
 }
 
 static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
+  if (pCfg->numOfVgroups < TSDB_MIN_VNODES_PER_DB || pCfg->numOfVgroups > TSDB_MAX_VNODES_PER_DB) return -1;
   if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) return -1;
   if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) return -1;
   if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1;
@@ -222,6 +222,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
 }
 
 static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
+  if (pCfg->numOfVgroups < 0) pCfg->numOfVgroups = TSDB_DEFAULT_VN_PER_DB;
   if (pCfg->cacheBlockSize < 0) pCfg->cacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
   if (pCfg->totalBlocks < 0) pCfg->totalBlocks = TSDB_DEFAULT_TOTAL_BLOCKS;
   if (pCfg->daysPerFile < 0) pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE;
@@ -246,7 +247,7 @@ static int32_t mndSetRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgOb
   if (pDbRaw == NULL || mndTransAppendRedolog(pTrans, pDbRaw) != 0) return -1;
   sdbSetRawStatus(pDbRaw, SDB_STATUS_CREATING);
 
-  for (int v = 0; v < pDb->numOfVgroups; ++v) {
+  for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) {
     SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v);
     if (pVgRaw == NULL || mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1;
     sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING);
@@ -260,7 +261,7 @@ static int32_t mndSetUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgOb
   if (pDbRaw == NULL || mndTransAppendUndolog(pTrans, pDbRaw) != 0) return -1;
   sdbSetRawStatus(pDbRaw, SDB_STATUS_DROPPED);
 
-  for (int v = 0; v < pDb->numOfVgroups; ++v) {
+  for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) {
     SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v);
     if (pVgRaw == NULL || mndTransAppendUndolog(pTrans, pVgRaw) != 0) return -1;
     sdbSetRawStatus(pVgRaw, SDB_STATUS_DROPPED);
@@ -274,7 +275,7 @@ static int32_t mndSetCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVg
   if (pDbRaw == NULL || mndTransAppendCommitlog(pTrans, pDbRaw) != 0) return -1;
   sdbSetRawStatus(pDbRaw, SDB_STATUS_READY);
 
-  for (int v = 0; v < pDb->numOfVgroups; ++v) {
+  for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) {
     SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v);
     if (pVgRaw == NULL || mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1;
     sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
@@ -284,10 +285,60 @@ static int32_t mndSetCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVg
 }
 
 static int32_t mndSetRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
+  for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) {
+    SVgObj *pVgroup = pVgroups + v;
+
+    for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
+      SVnodeGid *pVgid = pVgroup->vnodeGid + vn;
+      SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
+      if (pDnode == NULL) {
+        return -1;
+      }
+
+      SEpSet epset = mndGetDnodeEpset(pDnode);
+      mndReleaseDnode(pMnode, pDnode);
+
+      SCreateVnodeMsg *pMsg = mndBuildCreateVnodeMsg(pMnode, pDnode, pDb, pVgroup);
+      if (pMsg == NULL) {
+        return -1;
+      }
+
+      if (mndTransAppendRedoAction(pTrans, &epset, TSDB_MSG_TYPE_ALTER_VNODE_IN, sizeof(SCreateVnodeMsg), pMsg) != 0) {
+        free(pMsg);
+        return -1;
+      }
+    }
+  }
+
   return 0;
 }
 
 static int32_t mndSetUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
+  for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) {
+    SVgObj *pVgroup = pVgroups + v;
+
+    for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
+      SVnodeGid *pVgid = pVgroup->vnodeGid + vn;
+      SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
+      if (pDnode == NULL) {
+        return -1;
+      }
+
+      SEpSet epset = mndGetDnodeEpset(pDnode);
+      mndReleaseDnode(pMnode, pDnode);
+
+      SDropVnodeMsg *pMsg = mndBuildDropVnodeMsg(pMnode, pDnode, pDb, pVgroup);
+      if (pMsg == NULL) {
+        return -1;
+      }
+
+      if (mndTransAppendUndoAction(pTrans, &epset, TSDB_MSG_TYPE_DROP_VNODE_IN, sizeof(SDropVnodeMsg), pMsg) != 0) {
+        free(pMsg);
+        return -1;
+      }
+    }
+  }
+
   return 0;
 }
 
@@ -298,11 +349,11 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat
   dbObj.createdTime = taosGetTimestampMs();
   dbObj.updateTime = dbObj.createdTime;
   dbObj.uid = mndGenerateUid(dbObj.name, TSDB_FULL_DB_NAME_LEN);
-  dbObj.numOfVgroups = pCreate->numOfVgroups;
   dbObj.hashMethod = 1;
   dbObj.cfgVersion = 1;
   dbObj.vgVersion = 1;
-  dbObj.cfg = (SDbCfg){.cacheBlockSize = pCreate->cacheBlockSize,
+  dbObj.cfg = (SDbCfg){.numOfVgroups = pCreate->numOfVgroups,
+                       .cacheBlockSize = pCreate->cacheBlockSize,
                        .totalBlocks = pCreate->totalBlocks,
                        .daysPerFile = pCreate->daysPerFile,
                        .daysToKeep0 = pCreate->daysToKeep0,
@@ -643,7 +694,7 @@ static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) {
     return -1;
   }
 
-  int32_t contLen = sizeof(SUseDbRsp) + pDb->numOfVgroups * sizeof(SVgroupInfo);
+  int32_t    contLen = sizeof(SUseDbRsp) + pDb->cfg.numOfVgroups * sizeof(SVgroupInfo);
   SUseDbRsp *pRsp = rpcMallocCont(contLen);
   if (pRsp == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -654,7 +705,7 @@ static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) {
 
   if (pUse->vgVersion < pDb->vgVersion) {
     void *pIter = NULL;
-    while (vindex < pDb->numOfVgroups) {
+    while (vindex < pDb->cfg.numOfVgroups) {
       SVgObj *pVgroup = NULL;
       pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
       if (pIter == NULL) break;
@@ -888,7 +939,7 @@ static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    *(int16_t *)pWrite = pDb->numOfVgroups;
+    *(int16_t *)pWrite = pDb->cfg.numOfVgroups;
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;

source/dnode/mnode/impl/src/mndDnode.c

@@ -180,8 +180,12 @@ static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOldDnode, SDnodeObj
 }
 
 SDnodeObj *mndAcquireDnode(SMnode *pMnode, int32_t dnodeId) {
-  SSdb *pSdb = pMnode->pSdb;
-  return sdbAcquire(pSdb, SDB_DNODE, &dnodeId);
+  SSdb      *pSdb = pMnode->pSdb;
+  SDnodeObj *pDnode = sdbAcquire(pSdb, SDB_DNODE, &dnodeId);
+  if (pDnode == NULL) {
+    terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
+  }
+  return pDnode;
 }
 
 void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode) {

source/dnode/mnode/impl/src/mndTrans.c

@@ -21,6 +21,13 @@
 #define TSDB_TRN_ARRAY_SIZE 8
 #define TSDB_TRN_RESERVE_SIZE 64
 
+typedef struct {
+  SEpSet  epSet;
+  int8_t  msgType;
+  int32_t contLen;
+  void   *pCont;
+} STransAction;
+
 static SSdbRaw *mndTransActionEncode(STrans *pTrans);
 static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw);
 static int32_t  mndTransActionInsert(SSdb *pSdb, STrans *pTrans);
@@ -29,9 +36,12 @@ static int32_t  mndTransActionDelete(SSdb *pSdb, STrans *pTrans);
 
 static void    mndTransSetRpcHandle(STrans *pTrans, void *rpcHandle);
 static void    mndTransSendRpcRsp(STrans *pTrans, int32_t code);
-static int32_t mndTransAppendArray(SArray *pArray, SSdbRaw *pRaw);
-static void    mndTransDropArray(SArray *pArray);
-static int32_t mndTransExecuteArray(SMnode *pMnode, SArray *pArray);
+static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw);
+static int32_t mndTransAppendAction(SArray *pArray, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont);
+static void    mndTransDropLogs(SArray *pArray);
+static void    mndTransDropActions(SArray *pArray);
+static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray);
+static int32_t mndTransExecuteActions(SMnode *pMnode, SArray *pArray);
 static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans);
 static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans);
 static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans);
@@ -58,7 +68,7 @@ int32_t mndInitTrans(SMnode *pMnode) {
 void mndCleanupTrans(SMnode *pMnode) {}
 
 static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
-  int32_t rawDataLen = 16 * sizeof(int32_t) + TSDB_TRN_RESERVE_SIZE;
+  int32_t rawDataLen = sizeof(STrans) + TSDB_TRN_RESERVE_SIZE;
   int32_t redoLogNum = taosArrayGetSize(pTrans->redoLogs);
   int32_t undoLogNum = taosArrayGetSize(pTrans->undoLogs);
   int32_t commitLogNum = taosArrayGetSize(pTrans->commitLogs);
@@ -80,6 +90,16 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
     rawDataLen += sdbGetRawTotalSize(pTmp);
   }
 
+  for (int32_t i = 0; i < redoActionNum; ++i) {
+    STransAction *pAction = taosArrayGet(pTrans->redoActions, i);
+    rawDataLen += (sizeof(STransAction) + pAction->contLen);
+  }
+
+  for (int32_t i = 0; i < undoActionNum; ++i) {
+    STransAction *pAction = taosArrayGet(pTrans->undoActions, i);
+    rawDataLen += (sizeof(STransAction) + pAction->contLen);
+  }
+
   SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, TSDB_TRANS_VER, rawDataLen);
   if (pRaw == NULL) {
     mError("trans:%d, failed to alloc raw since %s", pTrans->id, terrstr());
@@ -116,6 +136,22 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
     SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len)
   }
 
+  for (int32_t i = 0; i < redoActionNum; ++i) {
+    STransAction *pAction = taosArrayGet(pTrans->redoActions, i);
+    SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet));
+    SDB_SET_INT8(pRaw, dataPos, pAction->msgType)
+    SDB_SET_INT32(pRaw, dataPos, pAction->contLen)
+    SDB_SET_BINARY(pRaw, dataPos, pAction->pCont, pAction->contLen);
+  }
+
+  for (int32_t i = 0; i < undoActionNum; ++i) {
+    STransAction *pAction = taosArrayGet(pTrans->undoActions, i);
+    SDB_SET_BINARY(pRaw, dataPos, (void *)&pAction->epSet, sizeof(SEpSet));
+    SDB_SET_INT8(pRaw, dataPos, pAction->msgType)
+    SDB_SET_INT32(pRaw, dataPos, pAction->contLen)
+    SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pCont, pAction->contLen);
+  }
+
   SDB_SET_RESERVE(pRaw, dataPos, TSDB_TRN_RESERVE_SIZE)
   SDB_SET_DATALEN(pRaw, dataPos);
   mTrace("trans:%d, encode to raw:%p, len:%d", pTrans->id, pRaw, dataPos);
@@ -147,8 +183,8 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
   pTrans->redoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
   pTrans->undoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
   pTrans->commitLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
-  pTrans->redoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
-  pTrans->undoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
+  pTrans->redoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(STransAction));
+  pTrans->undoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(STransAction));
 
   if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL ||
       pTrans->redoActions == NULL || pTrans->undoActions == NULL) {
@@ -175,42 +211,77 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
   for (int32_t i = 0; i < redoLogNum; ++i) {
     int32_t dataLen = 0;
     SDB_GET_INT32(pRaw, pRow, dataPos, &dataLen)
-
     char *pData = malloc(dataLen);
     SDB_GET_BINARY(pRaw, pRow, dataPos, pData, dataLen);
+
     void *ret = taosArrayPush(pTrans->redoLogs, &pData);
     if (ret == NULL) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto TRANS_DECODE_OVER;
-      break;
     }
   }
 
   for (int32_t i = 0; i < undoLogNum; ++i) {
     int32_t dataLen = 0;
     SDB_GET_INT32(pRaw, pRow, dataPos, &dataLen)
-
     char *pData = malloc(dataLen);
     SDB_GET_BINARY(pRaw, pRow, dataPos, pData, dataLen);
+
     void *ret = taosArrayPush(pTrans->undoLogs, &pData);
     if (ret == NULL) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto TRANS_DECODE_OVER;
-      break;
     }
   }
 
   for (int32_t i = 0; i < commitLogNum; ++i) {
     int32_t dataLen = 0;
     SDB_GET_INT32(pRaw, pRow, dataPos, &dataLen)
-
     char *pData = malloc(dataLen);
     SDB_GET_BINARY(pRaw, pRow, dataPos, pData, dataLen);
+
     void *ret = taosArrayPush(pTrans->commitLogs, &pData);
     if (ret == NULL) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto TRANS_DECODE_OVER;
-      break;
+    }
+  }
+
+  for (int32_t i = 0; i < redoActionNum; ++i) {
+    STransAction action = {0};
+    SDB_GET_BINARY(pRaw, pRow, dataPos, (void *)&action.epSet, sizeof(SEpSet));
+    SDB_GET_INT8(pRaw, pRow, dataPos, &action.msgType)
+    SDB_GET_INT32(pRaw, pRow, dataPos, &action.contLen)
+    action.pCont = malloc(action.contLen);
+    if (action.pCont == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto TRANS_DECODE_OVER;
+    }
+    SDB_GET_BINARY(pRaw, pRow, dataPos, action.pCont, action.contLen);
+
+    void *ret = taosArrayPush(pTrans->redoActions, &action);
+    if (ret == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto TRANS_DECODE_OVER;
+    }
+  }
+
+  for (int32_t i = 0; i < undoActionNum; ++i) {
+    STransAction action = {0};
+    SDB_GET_BINARY(pRaw, pRow, dataPos, (void *)&action.epSet, sizeof(SEpSet));
+    SDB_GET_INT8(pRaw, pRow, dataPos, &action.msgType)
+    SDB_GET_INT32(pRaw, pRow, dataPos, &action.contLen)
+    action.pCont = malloc(action.contLen);
+    if (action.pCont == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto TRANS_DECODE_OVER;
+    }
+    SDB_GET_BINARY(pRaw, pRow, dataPos, action.pCont, action.contLen);
+
+    void *ret = taosArrayPush(pTrans->undoActions, &action);
+    if (ret == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto TRANS_DECODE_OVER;
     }
   }
 
@@ -237,11 +308,11 @@ static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) {
 static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) {
   mTrace("trans:%d, perform delete action, stage:%s", pTrans->id, mndTransStageStr(pTrans->stage));
 
-  mndTransDropArray(pTrans->redoLogs);
-  mndTransDropArray(pTrans->undoLogs);
-  mndTransDropArray(pTrans->commitLogs);
-  mndTransDropArray(pTrans->redoActions);
-  mndTransDropArray(pTrans->undoActions);
+  mndTransDropLogs(pTrans->redoLogs);
+  mndTransDropLogs(pTrans->undoLogs);
+  mndTransDropLogs(pTrans->commitLogs);
+  mndTransDropActions(pTrans->redoActions);
+  mndTransDropActions(pTrans->undoActions);
 
   return 0;
 }
@@ -274,6 +345,8 @@ char *mndTransStageStr(ETrnStage stage) {
       return "rollback";
     case TRN_STAGE_RETRY:
       return "retry";
+    case TRN_STAGE_OVER:
+      return "stop";
     default:
       return "undefined";
   }
@@ -305,8 +378,8 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) {
   pTrans->redoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
   pTrans->undoLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
   pTrans->commitLogs = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
-  pTrans->redoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
-  pTrans->undoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(void *));
+  pTrans->redoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(STransAction));
+  pTrans->undoActions = taosArrayInit(TSDB_TRN_ARRAY_SIZE, sizeof(STransAction));
 
   if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL ||
       pTrans->redoActions == NULL || pTrans->undoActions == NULL) {
@@ -319,7 +392,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) {
   return pTrans;
 }
 
-static void mndTransDropArray(SArray *pArray) {
+static void mndTransDropLogs(SArray *pArray) {
   for (int32_t i = 0; i < pArray->size; ++i) {
     SSdbRaw *pRaw = taosArrayGetP(pArray, i);
     tfree(pRaw);
@@ -328,12 +401,21 @@ static void mndTransDropArray(SArray *pArray) {
   taosArrayDestroy(pArray);
 }
 
+static void mndTransDropActions(SArray *pArray) {
+  for (int32_t i = 0; i < pArray->size; ++i) {
+    STransAction *pAction = taosArrayGet(pArray, i);
+    free(pAction->pCont);
+  }
+
+  taosArrayDestroy(pArray);
+}
+
 void mndTransDrop(STrans *pTrans) {
-  mndTransDropArray(pTrans->redoLogs);
-  mndTransDropArray(pTrans->undoLogs);
-  mndTransDropArray(pTrans->commitLogs);
-  mndTransDropArray(pTrans->redoActions);
-  mndTransDropArray(pTrans->undoActions);
+  mndTransDropLogs(pTrans->redoLogs);
+  mndTransDropLogs(pTrans->undoLogs);
+  mndTransDropLogs(pTrans->commitLogs);
+  mndTransDropActions(pTrans->redoActions);
+  mndTransDropActions(pTrans->undoActions);
 
   mDebug("trans:%d, data:%p is dropped", pTrans->id, pTrans);
   tfree(pTrans);
@@ -344,7 +426,7 @@ static void mndTransSetRpcHandle(STrans *pTrans, void *rpcHandle) {
   mTrace("trans:%d, set rpc handle:%p", pTrans->id, rpcHandle);
 }
 
-static int32_t mndTransAppendArray(SArray *pArray, SSdbRaw *pRaw) {
+static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw) {
   if (pArray == NULL || pRaw == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
@@ -360,32 +442,44 @@ static int32_t mndTransAppendArray(SArray *pArray, SSdbRaw *pRaw) {
 }
 
 int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw) {
-  int32_t code = mndTransAppendArray(pTrans->redoLogs, pRaw);
+  int32_t code = mndTransAppendLog(pTrans->redoLogs, pRaw);
   mTrace("trans:%d, raw:%p append to redo logs, code:0x%x", pTrans->id, pRaw, code);
   return code;
 }
 
 int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw) {
-  int32_t code = mndTransAppendArray(pTrans->undoLogs, pRaw);
+  int32_t code = mndTransAppendLog(pTrans->undoLogs, pRaw);
   mTrace("trans:%d, raw:%p append to undo logs, code:0x%x", pTrans->id, pRaw, code);
   return code;
 }
 
 int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw) {
-  int32_t code = mndTransAppendArray(pTrans->commitLogs, pRaw);
+  int32_t code = mndTransAppendLog(pTrans->commitLogs, pRaw);
   mTrace("trans:%d, raw:%p append to commit logs, code:0x%x", pTrans->id, pRaw, code);
   return code;
 }
 
-int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, void *pMsg) {
-  int32_t code = mndTransAppendArray(pTrans->redoActions, pMsg);
-  mTrace("trans:%d, msg:%p append to redo actions", pTrans->id, pMsg);
+static int32_t mndTransAppendAction(SArray *pArray, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) {
+  STransAction action = {.epSet = *pEpSet, .msgType = msgType, .contLen = contLen, .pCont = pCont};
+
+  void *ptr = taosArrayPush(pArray, &action);
+  if (ptr == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  return 0;
+}
+
+int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) {
+  int32_t code = mndTransAppendAction(pTrans->redoActions, pEpSet, msgType, contLen, pCont);
+  mTrace("trans:%d, msg:%s len:%d append to redo actions", pTrans->id, taosMsg[msgType], contLen);
   return code;
 }
 
-int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, void *pMsg) {
-  int32_t code = mndTransAppendArray(pTrans->undoActions, pMsg);
-  mTrace("trans:%d, msg:%p append to undo actions", pTrans->id, pMsg);
+int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) {
+  int32_t code = mndTransAppendAction(pTrans->undoActions, pEpSet, msgType, contLen, pCont);
+  mTrace("trans:%d, msg:%s len:%d append to undo actions", pTrans->id, taosMsg[msgType], contLen);
   return code;
 }
 
@@ -502,7 +596,7 @@ void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code)
   // todo
 }
 
-static int32_t mndTransExecuteArray(SMnode *pMnode, SArray *pArray) {
+static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) {
   SSdb   *pSdb = pMnode->pSdb;
   int32_t arraySize = taosArrayGetSize(pArray);
 
@@ -520,7 +614,7 @@ static int32_t mndTransExecuteArray(SMnode *pMnode, SArray *pArray) {
 static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans) {
   int32_t code = 0;
   if (taosArrayGetSize(pTrans->redoLogs) != 0) {
-    code = mndTransExecuteArray(pMnode, pTrans->redoLogs);
+    code = mndTransExecuteLogs(pMnode, pTrans->redoLogs);
     if (code != 0) {
       mError("trans:%d, failed to execute redo logs since %s", pTrans->id, terrstr())
     } else {
@@ -534,7 +628,7 @@ static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans) {
 static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans) {
   int32_t code = 0;
   if (taosArrayGetSize(pTrans->undoLogs) != 0) {
-    code = mndTransExecuteArray(pMnode, pTrans->undoLogs);
+    code = mndTransExecuteLogs(pMnode, pTrans->undoLogs);
     if (code != 0) {
       mError("trans:%d, failed to execute undo logs since %s", pTrans->id, terrstr())
     } else {
@@ -548,7 +642,7 @@ static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans) {
 static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans) {
   int32_t code = 0;
   if (taosArrayGetSize(pTrans->commitLogs) != 0) {
-    code = mndTransExecuteArray(pMnode, pTrans->commitLogs);
+    code = mndTransExecuteLogs(pMnode, pTrans->commitLogs);
     if (code != 0) {
       mError("trans:%d, failed to execute commit logs since %s", pTrans->id, terrstr())
     } else {
@@ -559,18 +653,40 @@ static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans) {
   return code;
 }
 
-static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) {
-  if (taosArrayGetSize(pTrans->redoActions) != 0) {
-    mTrace("trans:%d, execute redo actions finished", pTrans->id);
+static int32_t mndTransExecuteActions(SMnode *pMnode, SArray *pArray) {
+#if 0
+  int32_t arraySize = taosArrayGetSize(pArray);
+  for (int32_t i = 0; i < arraySize; ++i) {
+    STransAction *pAction = taosArrayGet(pArray, i);
+
+    SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen};
+    rpcMsg.pCont = rpcMallocCont(pAction->contLen);
+    if (rpcMsg.pCont == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return -1;
+    }
+    memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen);
+    mndSendMsgToDnode(pMnode, &pAction->epSet, &rpcMsg);
   }
+
+  return TSDB_CODE_MND_ACTION_IN_PROGRESS;
+#else
   return 0;
+#endif
+}
+
+static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) {
+  if (taosArrayGetSize(pTrans->redoActions) <= 0) return 0;
+
+  mTrace("trans:%d, start to execute redo actions", pTrans->id);
+  return mndTransExecuteActions(pMnode, pTrans->redoActions);
 }
 
 static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) {
-  if (taosArrayGetSize(pTrans->undoActions) != 0) {
-    mTrace("trans:%d, execute undo actions finished", pTrans->id);
-  }
-  return 0;
+  if (taosArrayGetSize(pTrans->undoActions) <= 0) return 0;
+
+  mTrace("trans:%d, start to execute undo actions", pTrans->id);
+  return mndTransExecuteActions(pMnode, pTrans->undoActions);
 }
 
 static int32_t mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) {

source/dnode/mnode/impl/src/mndVgroup.c

@@ -24,9 +24,10 @@
 #define TSDB_VGROUP_VER_NUM 1
 #define TSDB_VGROUP_RESERVE_SIZE 64
 
-static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup);
-static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup);
-static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup);
+static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw);
+static int32_t  mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup);
+static int32_t  mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup);
+static int32_t  mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup);
 
 static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg);
 static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg);
@@ -156,9 +157,78 @@ void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) {
   sdbRelease(pSdb, pVgroup);
 }
 
-static int32_t mndGetDefaultVgroupSize(SMnode *pMnode) {
-  // todo
-  return 2;
+SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) {
+  SCreateVnodeMsg *pCreate = malloc(sizeof(SCreateVnodeMsg));
+  if (pCreate == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  pCreate->dnodeId = htonl(pDnode->id);
+  pCreate->vgId = htonl(pVgroup->vgId);
+  memcpy(pCreate->db, pDb->name, TSDB_FULL_DB_NAME_LEN);
+  pCreate->dbUid = htobe64(pDb->uid);
+  pCreate->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize);
+  pCreate->totalBlocks = htonl(pDb->cfg.totalBlocks);
+  pCreate->daysPerFile = htonl(pDb->cfg.daysPerFile);
+  pCreate->daysToKeep0 = htonl(pDb->cfg.daysToKeep0);
+  pCreate->daysToKeep1 = htonl(pDb->cfg.daysToKeep1);
+  pCreate->daysToKeep2 = htonl(pDb->cfg.daysToKeep2);
+  pCreate->minRows = htonl(pDb->cfg.minRows);
+  pCreate->maxRows = htonl(pDb->cfg.maxRows);
+  pCreate->commitTime = htonl(pDb->cfg.commitTime);
+  pCreate->fsyncPeriod = htonl(pDb->cfg.fsyncPeriod);
+  pCreate->walLevel = pDb->cfg.walLevel;
+  pCreate->precision = pDb->cfg.precision;
+  pCreate->compression = pDb->cfg.compression;
+  pCreate->quorum = pDb->cfg.quorum;
+  pCreate->update = pDb->cfg.update;
+  pCreate->cacheLastRow = pDb->cfg.cacheLastRow;
+  pCreate->replica = pVgroup->replica;
+  pCreate->selfIndex = -1;
+
+  for (int32_t v = 0; v < pVgroup->replica; ++v) {
+    SReplica  *pReplica = &pCreate->replicas[v];
+    SVnodeGid *pVgid = &pVgroup->vnodeGid[v];
+    SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
+    if (pVgidDnode == NULL) {
+      free(pCreate);
+      terrno = TSDB_CODE_MND_APP_ERROR;
+      return NULL;
+    }
+
+    pReplica->id = htonl(pVgidDnode->id);
+    pReplica->port = htons(pVgidDnode->port);
+    memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN);
+    mndReleaseDnode(pMnode, pVgidDnode);
+
+    if (pDnode->id == pVgid->dnodeId) {
+      pCreate->selfIndex = v;
+    }
+  }
+
+  if (pCreate->selfIndex == -1) {
+    free(pCreate);
+    terrno = TSDB_CODE_MND_APP_ERROR;
+    return NULL;
+  }
+
+  return pCreate;
+}
+
+SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) {
+  SDropVnodeMsg *pDrop = malloc(sizeof(SDropVnodeMsg));
+  if (pDrop == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  pDrop->dnodeId = htonl(pDnode->id);
+  pDrop->vgId = htonl(pVgroup->vgId);
+  memcpy(pDrop->db, pDb->name, TSDB_FULL_DB_NAME_LEN);
+  pDrop->dbUid = htobe64(pDb->uid);
+
+  return pDrop;
 }
 
 static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup) {
@@ -193,21 +263,7 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup) {
 }
 
 int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
-  if (pDb->numOfVgroups != -1 &&
-      (pDb->numOfVgroups < TSDB_MIN_VNODES_PER_DB || pDb->numOfVgroups > TSDB_MAX_VNODES_PER_DB)) {
-    terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
-    return -1;
-  }
-
-  if (pDb->numOfVgroups == -1) {
-    pDb->numOfVgroups = mndGetDefaultVgroupSize(pMnode);
-    if (pDb->numOfVgroups < 0) {
-      terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
-      return -1;
-    }
-  }
-
-  SVgObj *pVgroups = calloc(pDb->numOfVgroups, sizeof(SVgObj));
+  SVgObj *pVgroups = calloc(pDb->cfg.numOfVgroups, sizeof(SVgObj));
   if (pVgroups == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
@@ -217,9 +273,9 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
   int32_t  maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP);
   uint32_t hashMin = 0;
   uint32_t hashMax = UINT32_MAX;
-  uint32_t hashInterval = (hashMax - hashMin) / pDb->numOfVgroups;
+  uint32_t hashInterval = (hashMax - hashMin) / pDb->cfg.numOfVgroups;
 
-  for (uint32_t v = 0; v < pDb->numOfVgroups; v++) {
+  for (uint32_t v = 0; v < pDb->cfg.numOfVgroups; v++) {
     SVgObj *pVgroup = &pVgroups[v];
     pVgroup->vgId = maxVgId++;
     pVgroup->createdTime = taosGetTimestampMs();
@@ -227,7 +283,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
     pVgroup->version = 1;
     pVgroup->dbUid = pDb->uid;
     pVgroup->hashBegin = hashMin + hashInterval * v;
-    if (v == pDb->numOfVgroups - 1) {
+    if (v == pDb->cfg.numOfVgroups - 1) {
       pVgroup->hashEnd = hashMax;
     } else {
       pVgroup->hashEnd = hashMin + hashInterval * (v + 1) - 1;

source/libs/CMakeLists.txt

@@ -10,4 +10,4 @@ add_subdirectory(catalog)
 add_subdirectory(executor)
 add_subdirectory(planner)
 add_subdirectory(function)
-add_subdirectory(query)
+add_subdirectory(qcom)

source/libs/catalog/CMakeLists.txt

@@ -8,7 +8,7 @@ target_include_directories(
 
 target_link_libraries(
    catalog
-   PRIVATE os util common transport query
+   PRIVATE os util transport qcom
 )
 
 ADD_SUBDIRECTORY(test)
\ No newline at end of file

source/libs/catalog/test/CMakeLists.txt

@@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 ADD_EXECUTABLE(catalogTest ${SOURCE_LIST})
 TARGET_LINK_LIBRARIES(
         catalogTest
-        PUBLIC os util common catalog transport gtest query taos
+        PUBLIC os util common catalog transport gtest qcom taos
 )
 
 TARGET_INCLUDE_DIRECTORIES(

source/libs/index/CMakeLists.txt

@@ -9,6 +9,7 @@ target_link_libraries(
     index 
     PUBLIC os
     PUBLIC util
+    PUBLIC common
 )
 
 if (${BUILD_WITH_LUCENE})
@@ -21,9 +22,13 @@ if (${BUILD_WITH_LUCENE})
     index
     PUBLIC lucene++
   )
-
 endif(${BUILD_WITH_LUCENE})
 
+if (${BUILD_WITH_INVERTEDINDEX})
+  add_definitions(-DUSE_INVERTED_INDEX) 
+endif(${BUILD_WITH_INVERTEDINDEX})
+
+
 if (${BUILD_TEST})
   add_subdirectory(test)
 endif(${BUILD_TEST})

source/libs/index/inc/indexInt.h

@@ -37,9 +37,11 @@ struct SIndex {
 #endif  
  void     *cache;
  void     *tindex; 
- SHashObj *fieldObj;  // <field name, field id> 
- uint64_t suid; 
- int      fieldId; 
+ SHashObj *colObj;// < field name, field id> 
+ 
+ int64_t  suid;      // current super table id, -1 is normal table    
+ int      colId;  // field id allocated to cache
+ int32_t  cVersion; // current version allocated to cache 
  pthread_mutex_t mtx;
 };   
 
@@ -58,21 +60,21 @@ struct SIndexMultiTermQuery {
 
 // field and key;
 typedef struct SIndexTerm {
-  char    *key;
-  int32_t nKey;
-  char    *val;
-  int32_t nVal;
+  int64_t           suid; 
+  SIndexOperOnColumn operType; // oper type, add/del/update
+  uint8_t            colType;  // term data type, str/interger/json
+  char    *colName;
+  int32_t nColName;
+  char    *colVal;
+  int32_t nColVal;
 } SIndexTerm;
 
 typedef struct SIndexTermQuery {
-  SIndexTerm*     field_value;
-  EIndexQueryType type;
+  SIndexTerm*     term;
+  EIndexQueryType qType;
 } SIndexTermQuery;
 
 
-SIndexTerm *indexTermCreate(const char *key, int32_t nKey, const char *val, int32_t nVal);
-void        indexTermDestroy(SIndexTerm *p);
-
 
 #define indexFatal(...) do { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("index FATAL ", 255, __VA_ARGS__); }}     while(0)
 #define indexError(...) do { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("index ERROR ", 255, __VA_ARGS__); }}     while(0)

source/libs/index/inc/index_cache.h

@@ -17,11 +17,12 @@
 
 #include "index.h"
 #include "tlockfree.h"
+#include "tskiplist.h"
 // ----------------- row structure in skiplist ---------------------
 
 /* A data row, the format is like below:
- * |<--totalLen-->|<-- fieldId-->|<-- value len--->|<-- value-->|<--version--->|<-- itermType -->|
- *
+ * content: |<--totalLen-->|<-- fieldid-->|<--field type -->|<-- value len--->|<-- value -->|<--  uid  -->|<--version--->|<-- itermType -->|
+ * len    : |<--int32_t -->|<-- int16_t-->|<--  int16_t --->|<--- int32_t --->|<--valuelen->|<--uint64_t->|<-- int32_t-->|<--  int8_t  --->| 
  */
 
 #ifdef __cplusplus
@@ -30,19 +31,20 @@ extern "C" {
 
 typedef struct IndexCache {
   T_REF_DECLARE() 
-  int cVersion; // 
+  SSkipList *skiplist;
 } IndexCache;
 
 
 // 
 IndexCache *indexCacheCreate();
 
-void indexCacheDestroy(IndexCache *cache);
+void indexCacheDestroy(void *cache);
 
-int indexCachePut(IndexCache *cache, int32_t fieldId,  const char *fieldVale,  int32_t fvlen, uint64_t uid, int8_t operaType); 
+int indexCachePut(void *cache, int16_t fieldId, int16_t fieldType, const char *fieldValue,  int32_t fvLen, 
+              uint32_t version, uint64_t uid, int8_t operType);
 
-int indexCacheGet(IndexCache *cache, uint64_t *rst);
-int indexCacheSearch(IndexCache *cache, SIndexMultiTermQuery *query, SArray *result);
+int indexCacheGet(void *cache, uint64_t *rst);
+int indexCacheSearch(void *cache, SIndexMultiTermQuery *query, SArray *result);
 
 #ifdef __cplusplus
 }

source/libs/index/inc/index_fst.h

@@ -315,7 +315,6 @@ typedef struct StreamWithStateResult {
   FstSlice data;  
   FstOutput out;
   void     *state; 
-
 } StreamWithStateResult;
 
 StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *state);

source/libs/index/src/index.c

@@ -22,10 +22,10 @@
 #endif
 
 
-typedef struct SIdxFieldInfo {
-  int id;   // generated by index internal  
-  int type; // field type 
-} SIdxFieldInfo; 
+typedef struct SIdxColInfo {
+  int colId;     // generated by index internal  
+  int cVersion;
+} SIdxColInfo; 
 
 static pthread_once_t isInit = PTHREAD_ONCE_INIT;
 static void indexInit();
@@ -37,20 +37,25 @@ static int indexMergeCacheIntoTindex(struct SIndex *sIdx) {
   indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid); 
   return 0;
 }
-SIndex *indexOpen(SIndexOpts *opts, const char *path) {
+int indexOpen(SIndexOpts *opts, const char *path, SIndex **index) {
   pthread_once(&isInit, indexInit);
-  SIndex *sIdx = malloc(sizeof(SIndex));
+  SIndex *sIdx = calloc(1, sizeof(SIndex));
+  if (sIdx == NULL) { return -1; }
 
 #ifdef USE_LUCENE  
   index_t *index = index_open(path);      
   sIdx->index = index;
 #endif
-  
+
   sIdx->cache    = (void*)indexCacheCreate();     
   sIdx->tindex   = NULL;
-  sIdx->fieldObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+  sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+  sIdx->colId  = 1; 
+  sIdx->cVersion = 1; 
   pthread_mutex_init(&sIdx->mtx, NULL);
-  return sIdx;  
+
+  *index = sIdx; 
+  return 0;  
 }
 
 void indexClose(SIndex *sIdx) {
@@ -58,14 +63,17 @@ void indexClose(SIndex *sIdx) {
   index_close(sIdex->index); 
   sIdx->index = NULL;
 #endif
+
+#ifdef USE_INVERTED_INDEX
   indexCacheDestroy(sIdx->cache);
-  taosHashCleanup(sIdx->fieldObj); 
+  taosHashCleanup(sIdx->colObj); 
   pthread_mutex_destroy(&sIdx->mtx);
+#endif
   free(sIdx);  
   return;
 }
 
-int indexPut(SIndex *index, SArray* field_vals, int uid) {
+int indexPut(SIndex *index, SIndexMultiTerm * fVals, int uid) {
   
 #ifdef USE_LUCENE 
     index_document_t *doc = index_document_create(); 
@@ -73,8 +81,8 @@ int indexPut(SIndex *index, SArray* field_vals, int uid) {
     char buf[16] = {0};
     sprintf(buf, "%d", uid);
     
-    for (int i = 0; i < taosArrayGetSize(field_vals); i++) {
-      SIndexTerm *p = taosArrayGetP(field_vals, i); 
+    for (int i = 0; i < taosArrayGetSize(fVals); i++) {
+      SIndexTerm *p = taosArrayGetP(fVals, i); 
       index_document_add(doc, (const char *)(p->key), p->nKey, (const char *)(p->val), p->nVal, 1); 
     }
     index_document_add(doc, NULL, 0, buf, strlen(buf), 0);
@@ -82,10 +90,39 @@ int indexPut(SIndex *index, SArray* field_vals, int uid) {
     index_put(index->index, doc);
     index_document_destroy(doc);
 #endif
+
+#ifdef USE_INVERTED_INDEX
+  
+   //TODO(yihao): reduce the lock range 
   pthread_mutex_lock(&index->mtx);   
+  for (int i = 0; i < taosArrayGetSize(fVals); i++) {
+    SIndexTerm     *p = taosArrayGetP(fVals, i);
+    SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName);
+    if (fi == NULL) {
+      SIdxColInfo tfi = {.colId = index->colId};
+      index->cVersion++; 
+      index->colId++;
+      taosHashPut(index->colObj, p->colName, p->nColName, &tfi, sizeof(tfi)); 
+    } else {
+      //TODO, del  
+    }
+  }   
   pthread_mutex_unlock(&index->mtx);
-  return 1;
 
+  for (int i = 0; i < taosArrayGetSize(fVals); i++) {
+    SIndexTerm     *p = taosArrayGetP(fVals, i);
+    SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName);
+    assert(fi !=  NULL); 
+    int32_t colId = fi->colId; 
+    int32_t version = index->cVersion;
+    int ret = indexCachePut(index->cache, colId, p->colType, p->colVal, p->nColVal, version, uid, p->operType);     
+    if (ret != 0) {
+      return ret; 
+    }
+  }
+#endif
+
+  return 0;
 }
 int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result) {
 #ifdef USE_LUCENE 
@@ -122,16 +159,26 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result
   free(fields);
   free(keys);
   free(types);
+#endif
+
+#ifdef USE_INVERTED_INDEX
+  
 #endif
   return 1;
 }
 
 
 int indexDelete(SIndex *index, SIndexMultiTermQuery *query) {
+#ifdef USE_INVERTED_INDEX
+#endif
     
   return 1;
 }
-int indexRebuild(SIndex *index, SIndexOpts *opts);
+int indexRebuild(SIndex *index, SIndexOpts *opts) {
+#ifdef USE_INVERTED_INDEX
+#endif
+
+}
 
 
 SIndexOpts *indexOptsCreate() {
@@ -152,59 +199,61 @@ SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType opera) {
   SIndexMultiTermQuery *p = (SIndexMultiTermQuery *)malloc(sizeof(SIndexMultiTermQuery));
   if (p == NULL) { return NULL; }
   p->opera  = opera; 
-  p->query = taosArrayInit(1, sizeof(SIndexTermQuery)); 
+  p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); 
   return p;
 }
 void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery) {
   for (int i = 0; i < taosArrayGetSize(pQuery->query); i++) {
     SIndexTermQuery *p = (SIndexTermQuery *)taosArrayGet(pQuery->query, i);
-    indexTermDestroy(p->field_value);
+    indexTermDestroy(p->term);
   }
   taosArrayDestroy(pQuery->query);     
   free(pQuery);
 };
-int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, const char *field, int32_t nFields, const char *value, int32_t nValue, EIndexQueryType type){
-  SIndexTerm *t = indexTermCreate(field, nFields, value, nValue);  
-  if (t == NULL) {return -1;}
-  SIndexTermQuery q = {.type = type, .field_value = t};   
+int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType qType){
+  SIndexTermQuery q = {.qType = qType, .term = term};   
   taosArrayPush(pQuery->query, &q);
   return 0;
 }
 
 
-SIndexTerm *indexTermCreate(const char *key, int32_t nKey, const char *val, int32_t nVal) {
-  SIndexTerm *t = (SIndexTerm *)malloc(sizeof(SIndexTerm)); 
-  t->key  = (char *)calloc(nKey + 1, 1);
-  memcpy(t->key, key, nKey);
-  t->nKey = nKey;
+SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char *colName, int32_t nColName, const char *colVal, int32_t nColVal) {
+  SIndexTerm *t = (SIndexTerm *)calloc(1, (sizeof(SIndexTerm))); 
+  if (t == NULL) { return NULL; }
+
+  t->suid    = suid;
+  t->operType= oper; 
+  t->colType = colType;
+
+  t->colName  = (char *)calloc(1, nColName + 1);
+  memcpy(t->colName, colName, nColName);
+  t->nColName = nColName;
 
-  t->val = (char *)calloc(nVal + 1, 1);
-  memcpy(t->val, val, nVal);
-  t->nVal = nVal;
+  t->colVal = (char *)calloc(1, nColVal + 1);
+  memcpy(t->colVal, colVal, nColVal);
+  t->nColVal = nColVal;
   return t;
 }
 void indexTermDestroy(SIndexTerm *p) {
-  free(p->key);
-  free(p->val);
+  free(p->colName);
+  free(p->colVal);
   free(p);
 }  
 
-SArray *indexMultiTermCreate() {
+SIndexMultiTerm *indexMultiTermCreate() {
   return taosArrayInit(4, sizeof(SIndexTerm *)); 
 }
 
-int indexMultiTermAdd(SArray *array, const char *field, int32_t nField, const char *val, int32_t nVal) {
-   SIndexTerm *term = indexTermCreate(field,  nField, val, nVal);  
-   if (term == NULL) { return -1; }
-   taosArrayPush(array, &term);
+int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term) {
+   taosArrayPush(terms, &term);
    return 0;
 }
-void indexMultiTermDestroy(SArray *array) {
-  for (int32_t i = 0; i < taosArrayGetSize(array); i++) {
-    SIndexTerm *p = taosArrayGetP(array, i);
+void indexMultiTermDestroy(SIndexMultiTerm *terms) {
+  for (int32_t i = 0; i < taosArrayGetSize(terms); i++) {
+    SIndexTerm *p = taosArrayGetP(terms, i);
     indexTermDestroy(p);
   }
-  taosArrayDestroy(array);
+  taosArrayDestroy(terms);
 }
 
 void indexInit() {

source/libs/index/src/index_cache.c

@@ -14,12 +14,18 @@
  */
 
 #include "index_cache.h"
+#include "tcompare.h"
 
+#define MAX_INDEX_KEY_LEN 256// test only, change later
+
+static char*  getIndexKey(const void *pData) {
+  return NULL;  
+} 
 static int32_t compareKey(const void *l, const void *r) {
   char *lp = (char *)l;
   char *rp = (char *)r;
 
-  // skip total len
+  // skip total len, not compare
   int32_t ll, rl;  // len    
   memcpy(&ll, lp, sizeof(int32_t));
   memcpy(&rl, rp, sizeof(int32_t));
@@ -27,7 +33,7 @@ static int32_t compareKey(const void *l, const void *r) {
   rp += sizeof(int32_t);
   
   // compare field id
-  int32_t lf, rf; // field id
+  int16_t lf, rf; // field id
   memcpy(&lf, lp, sizeof(lf));
   memcpy(&rf, rp, sizeof(rf));
   if (lf != rf) {
@@ -36,14 +42,22 @@ static int32_t compareKey(const void *l, const void *r) {
   lp += sizeof(lf);
   rp += sizeof(rf);
 
-  // compare field value 
+  // compare field type
+  int16_t lft, rft; 
+  memcpy(&lft, lp, sizeof(lft));
+  memcpy(&rft, rp, sizeof(rft));
+  lp += sizeof(lft);
+  rp += sizeof(rft);
+  assert(rft == rft);
+  
+  // skip value len  
   int32_t lfl, rfl;
   memcpy(&lfl, lp, sizeof(lfl)); 
   memcpy(&rfl, rp, sizeof(rfl)); 
   lp += sizeof(lfl);
   rp += sizeof(rfl);
   
-  //refator later
+  // compare value 
   int32_t i, j;
   for (i = 0, j = 0; i < lfl && j < rfl; i++, j++) {
     if (lp[i] == rp[j]) { continue; }
@@ -54,66 +68,85 @@ static int32_t compareKey(const void *l, const void *r) {
   lp += lfl;
   rp += rfl; 
 
-  // compare version
+  // skip uid 
+  uint64_t lu, ru;
+  memcpy(&lu, lp, sizeof(lu)); 
+  memcpy(&ru, rp, sizeof(ru));
+  lp += sizeof(lu);
+  rp += sizeof(ru);
+  
+  // compare version, desc order
   int32_t lv, rv;
   memcpy(&lv, lp, sizeof(lv));
   memcpy(&rv, rp, sizeof(rv));
   if (lv != rv) {
     return lv > rv ? -1 : 1; 
-  }  
+  }   
   lp += sizeof(lv);
   rp += sizeof(rv);
+  // not care item type
 
-  
   return 0;  
   
 } 
 IndexCache *indexCacheCreate() {
   IndexCache *cache = calloc(1, sizeof(IndexCache));
+  cache->skiplist = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey);
   return cache;
+  
 }
 
-void indexCacheDestroy(IndexCache *cache) {
-  free(cache);
+void indexCacheDestroy(void *cache) {
+  IndexCache *pCache = cache; 
+  if (pCache == NULL) { return; } 
+  tSkipListDestroy(pCache->skiplist);
+  free(pCache);
 }
 
-int indexCachePut(IndexCache *cache, int32_t fieldId, const char *fieldValue,  int32_t fvlen, uint64_t uid, int8_t operType) {
+int indexCachePut(void *cache, int16_t fieldId, int16_t fieldType, const char *fieldValue,  int32_t fvLen, 
+              uint32_t version, uint64_t uid, int8_t operType) {
   if (cache == NULL) { return -1;} 
-  int32_t version = T_REF_INC(cache);     
 
-  int32_t total = sizeof(int32_t) + sizeof(fieldId) + 4 + fvlen  + sizeof(version) + sizeof(uid) + sizeof(operType); 
+  IndexCache *pCache = cache;
+
+  // encode data
+  int32_t total = sizeof(int32_t) + sizeof(fieldId) + sizeof(fieldType) + sizeof(fvLen) + fvLen  + sizeof(version) + sizeof(uid) + sizeof(operType); 
 
   char *buf = calloc(1, total); 
   char *p   = buf;
 
-  memcpy(buf, &total, sizeof(total)); 
-  total += total;
+  memcpy(p, &total, sizeof(total)); 
+  p += sizeof(total);
 
-  memcpy(buf, &fieldId, sizeof(fieldId));   
-  buf += sizeof(fieldId);
+  memcpy(p, &fieldId, sizeof(fieldId));   
+  p += sizeof(fieldId);
 
-  memcpy(buf, &fvlen, sizeof(fvlen));
-  buf += sizeof(fvlen);
-  memcpy(buf, fieldValue, fvlen); 
-  buf += fvlen;
+  memcpy(p, &fieldType, sizeof(fieldType));
+  p += sizeof(fieldType);
+  
+  memcpy(p, &fvLen, sizeof(fvLen));
+  p += sizeof(fvLen);
+  memcpy(p, fieldValue, fvLen); 
+  p += fvLen;
 
-  memcpy(buf, &version, sizeof(version));
-  buf += sizeof(version);
+  memcpy(p, &version, sizeof(version));
+  p += sizeof(version);
 
-  memcpy(buf, &uid, sizeof(uid));  
-  buf += sizeof(uid);
+  memcpy(p, &uid, sizeof(uid));  
+  p += sizeof(uid);
 
-  memcpy(buf, &operType, sizeof(operType));
-  buf += sizeof(operType); 
- 
+  memcpy(p, &operType, sizeof(operType));
+  p += sizeof(operType); 
 
+  tSkipListPut(pCache->skiplist, (void *)buf);  
+  // encode end
+    
 }
-int indexCacheSearch(IndexCache *cache, SIndexMultiTermQuery *query, SArray *result) {
-  
+int indexCacheDel(void *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) {
+  IndexCache *pCache = cache;
+  return 0;
+}
+int indexCacheSearch(void *cache, SIndexMultiTermQuery *query, SArray *result) {
+   
   return 0;  
 }
-
-
-
-
-

source/libs/index/test/CMakeLists.txt

 add_executable(indexTest "")
 target_sources(indexTest
   PRIVATE 
-  "indexTests.cpp" 
+  "indexTests.cc" 
 )
 target_include_directories ( indexTest
   PUBLIC

source/libs/index/test/indexTests.cpp → source/libs/index/test/indexTests.cc

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
 #include <gtest/gtest.h>
 #include <string>
 #include <iostream>
@@ -61,7 +75,7 @@ class FstReadMemory {
    // add later
    bool Search(AutomationCtx *ctx, std::vector<uint64_t> &result) { 
       FstStreamBuilder *sb = fstSearch(_fst, ctx);
-      StreamWithState *st = streamBuilderIntoStream(sb);  
+      StreamWithState  *st = streamBuilderIntoStream(sb);  
       StreamWithStateResult *rt = NULL;  
       
       while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
@@ -279,15 +293,71 @@ void validateFst() {
   delete m;
 } 
 
+class IndexEnv : public ::testing::Test {
+  protected:
+    virtual void SetUp() {
+      taosRemoveDir(path);  
+      opts = indexOptsCreate(); 
+      int ret = indexOpen(opts, path, &index); 
+      assert(ret == 0);
+    } 
+    virtual void TearDown() {
+      indexClose(index);
+      indexOptsDestroy(opts);
+    }
+    
+    const char *path = "/tmp/tindex";
+    SIndexOpts *opts;  
+    SIndex *index; 
+};
+
+TEST_F(IndexEnv, testPut) {
+
+   // single index column 
+   {
+     
+    std::string colName("tag1"), colVal("Hello world");
+    SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());  
+    SIndexMultiTerm *terms = indexMultiTermCreate();
+    indexMultiTermAdd(terms, term);
+   
+    for (size_t i = 0; i < 100; i++) {
+      int tableId = i;
+      int ret = indexPut(index, terms, tableId);
+      assert(ret == 0); 
+    }
+    indexMultiTermDestroy(terms);
+   }
+   // multi index column
+   {
+    
+    SIndexMultiTerm *terms = indexMultiTermCreate();
+    {
+      std::string colName("tag1"), colVal("Hello world");
+      SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());  
+      indexMultiTermAdd(terms, term);
+    }
+    {
+      std::string colName("tag2"), colVal("Hello world");
+      SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());  
+      indexMultiTermAdd(terms, term);
+    }
+    
+    for (int i = 0; i < 100; i++) {
+      int tableId = i;
+      int ret = indexPut(index, terms, tableId);
+      assert(ret == 0); 
+    } 
+    indexMultiTermDestroy(terms);
+   }
+   //   
+} 
 
-int main(int argc, char** argv) {
-  checkFstPerf(); 
-  //checkFstPrefixSearch();
-  return 1;
+TEST_F(IndexEnv, testDel) {
+     
 }
 
-//TEST(IndexFstBuilder, IndexFstInput) {
-//
-//}
+
+
 
 

source/libs/parser/CMakeLists.txt

@@ -8,7 +8,7 @@ target_include_directories(
 
 target_link_libraries(
         parser
-        PRIVATE os util common catalog function transport query
+        PRIVATE os util catalog function transport qcom
 )
 
 ADD_SUBDIRECTORY(test)

source/libs/parser/inc/astGenerator.h

@@ -171,8 +171,8 @@ typedef struct SCreateDbInfo {
   int8_t             update;
   int8_t             cachelast;
   SArray            *keep;
-  int8_t             dbType;
-  int16_t            partitions;
+//  int8_t             dbType;
+//  int16_t            partitions;
 } SCreateDbInfo;
 
 typedef struct SCreateFuncInfo {

source/libs/parser/inc/astToMsg.h

@@ -5,5 +5,7 @@
 #include "taosmsg.h"
 
 SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen);
+SShowMsg* buildShowMsg(SShowInfo* pShowInfo, int64_t id, char* msgBuf, int32_t msgLen);
+SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, char* msgBuf, int32_t msgLen);
 
 #endif  // TDENGINE_ASTTOMSG_H

source/libs/parser/inc/parserUtil.h

@@ -46,6 +46,7 @@ SInternalField* getInternalField(SFieldInfo* pFieldInfo, int32_t index);
 
 int32_t parserValidateIdToken(SToken* pToken);
 int32_t parserValidatePassword(SToken* pToken, SMsgBuf* pMsgBuf);
+int32_t parserValidateNameToken(SToken* pToken);
 
 int32_t buildInvalidOperationMsg(SMsgBuf* pMsgBuf, const char* msg);
 int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo,  const char* sourceStr);

source/libs/parser/src/astGenerator.c

@@ -948,27 +948,24 @@ void setCompactVnodeSql(SSqlInfo *pInfo, int32_t type, SArray *pParam) {
 void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo) {
   pDBInfo->compressionLevel = -1;
 
-  pDBInfo->walLevel = -1;
+  pDBInfo->walLevel    = -1;
   pDBInfo->fsyncPeriod = -1;
-  pDBInfo->commitTime = -1;
+  pDBInfo->commitTime  = -1;
   pDBInfo->maxTablesPerVnode = -1;
 
-  pDBInfo->cacheBlockSize = -1;
-  pDBInfo->numOfBlocks = -1;
+  pDBInfo->cacheBlockSize  = -1;
+  pDBInfo->numOfBlocks     = -1;
   pDBInfo->maxRowsPerBlock = -1;
   pDBInfo->minRowsPerBlock = -1;
-  pDBInfo->daysPerFile = -1;
+  pDBInfo->daysPerFile     = -1;
 
   pDBInfo->replica = -1;
-  pDBInfo->quorum = -1;
-  pDBInfo->keep = NULL;
+  pDBInfo->quorum  = -1;
+  pDBInfo->keep    = NULL;
 
-  pDBInfo->update = -1;
+  pDBInfo->update  = -1;
   pDBInfo->cachelast = -1;
 
-  pDBInfo->dbType = -1;
-  pDBInfo->partitions = -1;
-
   memset(&pDBInfo->precision, 0, sizeof(SToken));
 }
 

source/libs/parser/src/astToMsg.c

 #include "parserInt.h"
+#include "parserUtil.h"
 
 SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen) {
-  SCreateUserMsg *pMsg = (SCreateUserMsg *)calloc(1, sizeof(SCreateUserMsg));
+  SCreateUserMsg* pMsg = (SCreateUserMsg*)calloc(1, sizeof(SCreateUserMsg));
   if (pMsg == NULL) {
-//    tscError("0x%" PRIx64 " failed to malloc for query msg", id);
+    //    tscError("0x%" PRIx64 " failed to malloc for query msg", id);
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return NULL;
   }
 
-  SUserInfo *pUser = &pInfo->pMiscInfo->user;
+  SUserInfo* pUser = &pInfo->pMiscInfo->user;
   strncpy(pMsg->user, pUser->user.z, pUser->user.n);
   pMsg->type = pUser->type;
   pMsg->superUser = (int8_t)pUser->type;
 
   if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) {
-//    pMsg->privilege = (char)pCmd->count;
+    //    pMsg->privilege = (char)pCmd->count;
   } else {
     strncpy(pMsg->pass, pUser->passwd.z, pUser->passwd.n);
   }
 
   return pMsg;
-}
+}
\ No newline at end of file
+
+SShowMsg* buildShowMsg(SShowInfo* pShowInfo, int64_t id, char* msgBuf, int32_t msgLen) {
+  SShowMsg* pShowMsg = calloc(1, sizeof(SShowMsg));
+
+  pShowMsg->type = pShowInfo->showType;
+
+  if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) {
+    SToken* pPattern = &pShowInfo->pattern;
+    if (pPattern->type > 0) {  // only show tables support wildcard query
+      strncpy(pShowMsg->payload, pPattern->z, pPattern->n);
+      pShowMsg->payloadLen = htons(pPattern->n);
+    }
+  } else {
+    SToken* pEpAddr = &pShowInfo->prefix;
+    assert(pEpAddr->n > 0 && pEpAddr->type > 0);
+
+    strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n);
+    pShowMsg->payloadLen = htons(pEpAddr->n);
+  }
+
+  return pShowMsg;
+}
+
+static int32_t setKeepOption(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDb, SMsgBuf* pMsgBuf) {
+  const char* msg1 = "invalid number of keep options";
+  const char* msg2 = "invalid keep value";
+  const char* msg3 = "invalid keep value, should be keep0 <= keep1 <= keep2";
+
+  pMsg->daysToKeep0 = htonl(-1);
+  pMsg->daysToKeep1 = htonl(-1);
+  pMsg->daysToKeep2 = htonl(-1);
+
+  SArray* pKeep = pCreateDb->keep;
+  if (pKeep != NULL) {
+    size_t s = taosArrayGetSize(pKeep);
+#ifdef _STORAGE
+    if (s >= 4 ||s <= 0) {
+#else
+    if (s != 1) {
+#endif
+      return buildInvalidOperationMsg(pMsgBuf, msg1);
+    }
+
+//    tListI* p0 = taosArrayGet(pKeep, 0);
+//    tVariantListItem* p1 = (s > 1) ? taosArrayGet(pKeep, 1) : p0;
+//    tVariantListItem* p2 = (s > 2) ? taosArrayGet(pKeep, 2) : p1;
+//
+//    if ((int32_t)p0->pVar.i64 <= 0 || (int32_t)p1->pVar.i64 <= 0 || (int32_t)p2->pVar.i64 <= 0) {
+//      return buildInvalidOperationMsg(pMsgBuf, msg2);
+//    }
+//    if (!(((int32_t)p0->pVar.i64 <= (int32_t)p1->pVar.i64) && ((int32_t)p1->pVar.i64 <= (int32_t)p2->pVar.i64))) {
+//      return buildInvalidOperationMsg(pMsgBuf, msg3);
+//    }
+//
+//    pMsg->daysToKeep0 = htonl((int32_t)p0->pVar.i64);
+//    pMsg->daysToKeep1 = htonl((int32_t)p1->pVar.i64);
+//    pMsg->daysToKeep2 = htonl((int32_t)p2->pVar.i64);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t setTimePrecision(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDbInfo, SMsgBuf* pMsgBuf) {
+  const char* msg = "invalid time precision";
+
+  pMsg->precision = TSDB_TIME_PRECISION_MILLI;  // millisecond by default
+
+  SToken* pToken = &pCreateDbInfo->precision;
+  if (pToken->n > 0) {
+    pToken->n = strdequote(pToken->z);
+
+    if (strncmp(pToken->z, TSDB_TIME_PRECISION_MILLI_STR, pToken->n) == 0 &&
+        strlen(TSDB_TIME_PRECISION_MILLI_STR) == pToken->n) {
+      // time precision for this db: million second
+      pMsg->precision = TSDB_TIME_PRECISION_MILLI;
+    } else if (strncmp(pToken->z, TSDB_TIME_PRECISION_MICRO_STR, pToken->n) == 0 &&
+               strlen(TSDB_TIME_PRECISION_MICRO_STR) == pToken->n) {
+      pMsg->precision = TSDB_TIME_PRECISION_MICRO;
+    } else if (strncmp(pToken->z, TSDB_TIME_PRECISION_NANO_STR, pToken->n) == 0 &&
+               strlen(TSDB_TIME_PRECISION_NANO_STR) == pToken->n) {
+      pMsg->precision = TSDB_TIME_PRECISION_NANO;
+    } else {
+      return buildInvalidOperationMsg(pMsgBuf, msg);
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static void doSetDbOptions(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDb) {
+  pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize);
+  pMsg->totalBlocks  = htonl(pCreateDb->numOfBlocks);
+  pMsg->daysPerFile  = htonl(pCreateDb->daysPerFile);
+  pMsg->commitTime   = htonl((int32_t)pCreateDb->commitTime);
+  pMsg->minRowsPerFileBlock = htonl(pCreateDb->minRowsPerBlock);
+  pMsg->maxRowsPerFileBlock = htonl(pCreateDb->maxRowsPerBlock);
+  pMsg->fsyncPeriod  = htonl(pCreateDb->fsyncPeriod);
+  pMsg->compression  = pCreateDb->compressionLevel;
+  pMsg->walLevel     = (char)pCreateDb->walLevel;
+  pMsg->replications = pCreateDb->replica;
+  pMsg->quorum       = pCreateDb->quorum;
+  pMsg->ignoreExist  = pCreateDb->ignoreExists;
+  pMsg->update       = pCreateDb->update;
+  pMsg->cacheLastRow = pCreateDb->cachelast;
+}
+
+int32_t setDbOptions(SCreateDbMsg* pCreateDbMsg, const SCreateDbInfo* pCreateDbSql, SMsgBuf* pMsgBuf) {
+  doSetDbOptions(pCreateDbMsg, pCreateDbSql);
+
+  if (setKeepOption(pCreateDbMsg, pCreateDbSql, pMsgBuf) != TSDB_CODE_SUCCESS) {
+    return TSDB_CODE_TSC_INVALID_OPERATION;
+  }
+
+  if (setTimePrecision(pCreateDbMsg, pCreateDbSql, pMsgBuf) != TSDB_CODE_SUCCESS) {
+    return TSDB_CODE_TSC_INVALID_OPERATION;
+  }
+
+  // todo configurable
+  pCreateDbMsg->numOfVgroups = htonl(2);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, char* msgBuf, int32_t msgLen) {
+  SCreateDbMsg* pCreateMsg = calloc(1, sizeof(SCreateDbMsg));
+
+  SMsgBuf msg = {.buf = msgBuf, .len = msgLen};
+  if (setDbOptions(pCreateMsg, pCreateDbInfo, &msg) != TSDB_CODE_SUCCESS) {
+    tfree(pCreateMsg);
+    terrno = TSDB_CODE_TSC_INVALID_OPERATION;
+
+    return NULL;
+  }
+
+  return pCreateMsg;
+}

source/libs/parser/src/astValidate.c

@@ -759,11 +759,6 @@ int32_t validateIntervalNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMs
   // It is a time window query
   pQueryInfo->info.timewindow = true;
   return TSDB_CODE_SUCCESS;
-  // disable it temporarily
-//  bool interpQuery = tscIsPointInterpQuery(pQueryInfo);
-//  if ((pSqlNode->interval.token == TK_EVERY && (!interpQuery)) || (pSqlNode->interval.token == TK_INTERVAL && interpQuery)) {
-//    return buildInvalidOperationMsg(pMsgBuf, msg4);
-//  }
 }
 
 int32_t validateSessionNode(SQueryStmtInfo *pQueryInfo, SSessionWindowVal* pSession, int32_t precision, SMsgBuf* pMsgBuf) {
@@ -3707,14 +3702,6 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
       return TSDB_CODE_SUCCESS;
     }
 
-    case TSDB_SQL_SHOW: {
-      if (setShowInfo(pSql, pInfo) != TSDB_CODE_SUCCESS) {
-        return TSDB_CODE_TSC_INVALID_OPERATION;
-      }
-
-      break;
-    }
-
     case TSDB_SQL_CREATE_FUNCTION:
     case TSDB_SQL_DROP_FUNCTION:  {
       code = handleUserDefinedFunc(pSql, pInfo);
@@ -3725,35 +3712,6 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
       break;
     }
 
-    case TSDB_SQL_ALTER_DB:
-    case TSDB_SQL_CREATE_DB: {
-      const char* msg1 = "invalid db name";
-      const char* msg2 = "name too long";
-
-      SCreateDbInfo* pCreateDB = &(pInfo->pMiscInfo->dbOpt);
-      if (pCreateDB->dbname.n >= TSDB_DB_NAME_LEN) {
-        return buildInvalidOperationMsg(pMsgBuf, msg2);
-      }
-
-      char buf[TSDB_DB_NAME_LEN] = {0};
-      SToken token = taosTokenDup(&pCreateDB->dbname, buf, tListLen(buf));
-
-      if (tscValidateName(&token) != TSDB_CODE_SUCCESS) {
-        return buildInvalidOperationMsg(pMsgBuf, msg1);
-      }
-
-      int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), &token);
-      if (ret != TSDB_CODE_SUCCESS) {
-        return buildInvalidOperationMsg(pMsgBuf, msg2);
-      }
-
-      if (parseCreateDBOptions(pCmd, pCreateDB) != TSDB_CODE_SUCCESS) {
-        return TSDB_CODE_TSC_INVALID_OPERATION;
-      }
-
-      break;
-    }
-
     case TSDB_SQL_CREATE_DNODE: {
       const char* msg = "invalid host name (ip address)";
 
@@ -4133,25 +4091,83 @@ static int32_t setShowInfo(struct SSqlInfo* pInfo, void** output, int32_t* msgLe
     }
   }
 
-  SShowMsg* pShowMsg = calloc(1, sizeof(SShowMsg));
-  pShowMsg->type = pShowInfo->showType;
+  *output = buildShowMsg(pShowInfo, 0, pMsgBuf->buf, pMsgBuf->len);
+  *msgLen = sizeof(SShowMsg)/* + htons(pShowMsg->payloadLen)*/;
+  return TSDB_CODE_SUCCESS;
+}
+
+// can only perform the parameters based on the macro definitation
+static int32_t doCheckDbOptions(SCreateDbMsg* pCreate, SMsgBuf* pMsgBuf) {
+  char msg[512] = {0};
 
-  if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) {
-    SToken* pPattern = &pShowInfo->pattern;
-    if (pPattern->type > 0) {  // only show tables support wildcard query
-      strncpy(pShowMsg->payload, pPattern->z, pPattern->n);
-      pShowMsg->payloadLen = htons(pPattern->n);
-    }
-  } else {
-    SToken* pEpAddr = &pShowInfo->prefix;
-    assert(pEpAddr->n > 0 && pEpAddr->type > 0);
+  if (pCreate->walLevel != -1 && (pCreate->walLevel < TSDB_MIN_WAL_LEVEL || pCreate->walLevel > TSDB_MAX_WAL_LEVEL)) {
+    snprintf(msg, tListLen(msg), "invalid db option walLevel: %d, only 1-2 allowed", pCreate->walLevel);
+    return buildInvalidOperationMsg(pMsgBuf, msg);
+  }
 
-    strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n);
-    pShowMsg->payloadLen = htons(pEpAddr->n);
+  if (pCreate->replications != -1 &&
+      (pCreate->replications < TSDB_MIN_DB_REPLICA_OPTION || pCreate->replications > TSDB_MAX_DB_REPLICA_OPTION)) {
+    snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications,
+             TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION);
+    return buildInvalidOperationMsg(pMsgBuf, msg);
+  }
+
+  int32_t blocks = ntohl(pCreate->totalBlocks);
+  if (blocks != -1 && (blocks < TSDB_MIN_TOTAL_BLOCKS || blocks > TSDB_MAX_TOTAL_BLOCKS)) {
+    snprintf(msg, tListLen(msg), "invalid db option totalBlocks: %d valid range: [%d, %d]", blocks,
+             TSDB_MIN_TOTAL_BLOCKS, TSDB_MAX_TOTAL_BLOCKS);
+    return buildInvalidOperationMsg(pMsgBuf, msg);
+  }
+
+  if (pCreate->quorum != -1 &&
+      (pCreate->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCreate->quorum > TSDB_MAX_DB_QUORUM_OPTION)) {
+    snprintf(msg, tListLen(msg), "invalid db option quorum: %d valid range: [%d, %d]", pCreate->quorum,
+             TSDB_MIN_DB_QUORUM_OPTION, TSDB_MAX_DB_QUORUM_OPTION);
+    return buildInvalidOperationMsg(pMsgBuf, msg);
+  }
+
+  int32_t val = htonl(pCreate->daysPerFile);
+  if (val != -1 && (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE)) {
+    snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val,
+             TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE);
+    return buildInvalidOperationMsg(pMsgBuf, msg);
+  }
+
+  val = htonl(pCreate->cacheBlockSize);
+  if (val != -1 && (val < TSDB_MIN_CACHE_BLOCK_SIZE || val > TSDB_MAX_CACHE_BLOCK_SIZE)) {
+    snprintf(msg, tListLen(msg), "invalid db option cacheBlockSize: %d valid range: [%d, %d]", val,
+             TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE);
+    return buildInvalidOperationMsg(pMsgBuf, msg);
+  }
+
+  if (pCreate->precision != TSDB_TIME_PRECISION_MILLI && pCreate->precision != TSDB_TIME_PRECISION_MICRO &&
+      pCreate->precision != TSDB_TIME_PRECISION_NANO) {
+    snprintf(msg, tListLen(msg), "invalid db option timePrecision: %d valid value: [%d, %d, %d]", pCreate->precision,
+             TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO);
+    return buildInvalidOperationMsg(pMsgBuf, msg);
+  }
+
+  val = htonl(pCreate->commitTime);
+  if (val != -1 && (val < TSDB_MIN_COMMIT_TIME || val > TSDB_MAX_COMMIT_TIME)) {
+    snprintf(msg, tListLen(msg), "invalid db option commitTime: %d valid range: [%d, %d]", val,
+             TSDB_MIN_COMMIT_TIME, TSDB_MAX_COMMIT_TIME);
+    return buildInvalidOperationMsg(pMsgBuf, msg);
+  }
+
+  val = htonl(pCreate->fsyncPeriod);
+  if (val != -1 && (val < TSDB_MIN_FSYNC_PERIOD || val > TSDB_MAX_FSYNC_PERIOD)) {
+    snprintf(msg, tListLen(msg), "invalid db option fsyncPeriod: %d valid range: [%d, %d]", val,
+             TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD);
+    return buildInvalidOperationMsg(pMsgBuf, msg);
+  }
+
+  if (pCreate->compression != -1 &&
+      (pCreate->compression < TSDB_MIN_COMP_LEVEL || pCreate->compression > TSDB_MAX_COMP_LEVEL)) {
+    snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression,
+             TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL);
+    return buildInvalidOperationMsg(pMsgBuf, msg);
   }
 
-  *output = pShowMsg;
-  *msgLen = sizeof(SShowMsg) + htons(pShowMsg->payloadLen);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -4216,6 +4232,36 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, in
       code = setShowInfo(pInfo, output, outputLen, pMsgBuf);
       break;
     }
+
+    case TSDB_SQL_ALTER_DB:
+    case TSDB_SQL_CREATE_DB: {
+      const char* msg1 = "invalid db name";
+      const char* msg2 = "name too long";
+
+      SCreateDbInfo* pCreateDB = &(pInfo->pMiscInfo->dbOpt);
+      if (pCreateDB->dbname.n >= TSDB_DB_NAME_LEN) {
+        return buildInvalidOperationMsg(pMsgBuf, msg2);
+      }
+
+      char buf[TSDB_DB_NAME_LEN] = {0};
+      SToken token = taosTokenDup(&pCreateDB->dbname, buf, tListLen(buf));
+
+      if (parserValidateNameToken(&token) != TSDB_CODE_SUCCESS) {
+        return buildInvalidOperationMsg(pMsgBuf, msg1);
+      }
+
+      SCreateDbMsg* pCreateMsg = buildCreateDbMsg(pCreateDB, pMsgBuf->buf, pMsgBuf->len);
+      if (doCheckDbOptions(pCreateMsg, pMsgBuf) != TSDB_CODE_SUCCESS) {
+        return TSDB_CODE_TSC_INVALID_OPERATION;
+      }
+
+      strncpy(pCreateMsg->db, token.z, token.n);
+
+      *output = pCreateMsg;
+      *outputLen = sizeof(SCreateDbMsg);
+      break;
+    }
+
     default:
       break;
   }

source/libs/parser/src/parser.c

@@ -77,7 +77,7 @@ static int32_t tnameComparFn(const void* p1, const void* p2) {
   SName* pn1 = (SName*)p1;
   SName* pn2 = (SName*)p2;
 
-  int32_t ret = strncmp(pn1->acctId, pn2->acctId, tListLen(pn1->acctId));
+  int32_t ret = pn1->acctId - pn2->acctId;
   if (ret != 0) {
     return ret > 0? 1:-1;
   } else {

source/libs/parser/src/parserUtil.c

@@ -122,6 +122,25 @@ int32_t parserValidatePassword(SToken* pToken, SMsgBuf* pMsgBuf) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t parserValidateNameToken(SToken* pToken) {
+  if (pToken == NULL || pToken->z == NULL || pToken->type != TK_ID) {
+    return TSDB_CODE_TSC_INVALID_OPERATION;
+  }
+
+  // it is a token quoted with escape char '`'
+  if (pToken->z[0] == TS_ESCAPE_CHAR && pToken->z[pToken->n - 1] == TS_ESCAPE_CHAR) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  char* sep = strnchr(pToken->z, TS_PATH_DELIMITER[0], pToken->n, true);
+  if (sep != NULL) {  // It is a complex type, not allow
+    return TSDB_CODE_TSC_INVALID_OPERATION;
+  }
+
+  strntolower(pToken->z, pToken->z, pToken->n);
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t buildInvalidOperationMsg(SMsgBuf* pBuf, const char* msg) {
   strncpy(pBuf->buf, msg, pBuf->len);
   return TSDB_CODE_TSC_INVALID_OPERATION;

source/libs/parser/src/sql.c

@@ -2519,7 +2519,7 @@ static void yy_reduce(
 { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; }
         break;
       case 105: /* db_optr ::= */
-{setDefaultCreateDbOption(&yymsp[1].minor.yy256); yymsp[1].minor.yy256.dbType = TSDB_DB_TYPE_DEFAULT;}
+{setDefaultCreateDbOption(&yymsp[1].minor.yy256);}
         break;
       case 106: /* db_optr ::= db_optr cache */
 { yylhsminor.yy256 = yymsp[-1].minor.yy256; yylhsminor.yy256.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
@@ -2590,16 +2590,16 @@ static void yy_reduce(
         break;
       case 121: /* topic_optr ::= db_optr */
       case 131: /* alter_topic_optr ::= alter_db_optr */ yytestcase(yyruleno==131);
-{ yylhsminor.yy256 = yymsp[0].minor.yy256; yylhsminor.yy256.dbType = TSDB_DB_TYPE_TOPIC; }
+{ yylhsminor.yy256 = yymsp[0].minor.yy256;}
   yymsp[0].minor.yy256 = yylhsminor.yy256;
         break;
       case 122: /* topic_optr ::= topic_optr partitions */
       case 132: /* alter_topic_optr ::= alter_topic_optr partitions */ yytestcase(yyruleno==132);
-{ yylhsminor.yy256 = yymsp[-1].minor.yy256; yylhsminor.yy256.partitions = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
+{ yylhsminor.yy256 = yymsp[-1].minor.yy256; }
   yymsp[-1].minor.yy256 = yylhsminor.yy256;
         break;
       case 123: /* alter_db_optr ::= */
-{ setDefaultCreateDbOption(&yymsp[1].minor.yy256); yymsp[1].minor.yy256.dbType = TSDB_DB_TYPE_DEFAULT;}
+{ setDefaultCreateDbOption(&yymsp[1].minor.yy256); }
         break;
       case 133: /* typename ::= ids */
 { 

source/libs/parser/test/CMakeLists.txt

@@ -15,7 +15,7 @@ TARGET_INCLUDE_DIRECTORIES(
 
 TARGET_LINK_LIBRARIES(
         parserTest
-        PUBLIC os util common parser catalog transport gtest function planner query
+        PUBLIC os util common parser catalog transport gtest function planner qcom
 )
 
 TARGET_LINK_OPTIONS(parserTest PRIVATE -Wl,-wrap,malloc)

source/libs/planner/CMakeLists.txt

@@ -8,7 +8,7 @@ target_include_directories(
 
 target_link_libraries(
     planner
-    PRIVATE os util common cjson catalog parser transport function query
+    PRIVATE os util catalog cjson parser transport function qcom
 )
 
 ADD_SUBDIRECTORY(test)

source/libs/planner/inc/plannerInt.h

@@ -100,8 +100,9 @@ int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str);
 int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql);
 
 int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag);
-
+int32_t setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SArray* eps);
 int32_t subPlanToString(const SSubplan *pPhyNode, char** str);
+int32_t stringToSubplan(const char* str, SSubplan** subplan);
 
 /**
  * Destroy the query plan object.
@@ -116,6 +117,8 @@ void destroyQueryPlan(struct SQueryPlanNode* pQueryNode);
  */
 void* destroyQueryPhyPlan(struct SPhyNode* pQueryPhyNode);
 
+int32_t opNameToOpType(const char* name);
+
 #ifdef __cplusplus
 }
 #endif

source/libs/planner/src/physicalPlan.c

@@ -19,6 +19,13 @@
 #define STORE_CURRENT_SUBPLAN(cxt) SSubplan* _ = cxt->pCurrentSubplan
 #define RECOVERY_CURRENT_SUBPLAN(cxt) cxt->pCurrentSubplan = _
 
+typedef struct SPlanContext {
+  struct SCatalog* pCatalog;
+  struct SQueryDag* pDag;
+  SSubplan* pCurrentSubplan;
+  SSubplanId nextId;
+} SPlanContext;
+
 static const char* gOpName[] = {
   "Unknown",
 #define INCLUDE_AS_NAME
@@ -26,12 +33,14 @@ static const char* gOpName[] = {
 #undef INCLUDE_AS_NAME
 };
 
-typedef struct SPlanContext {
-  struct SCatalog* pCatalog;
-  struct SQueryDag* pDag;
-  SSubplan* pCurrentSubplan;
-  SSubplanId nextId;
-} SPlanContext;
+int32_t opNameToOpType(const char* name) {
+  for (int32_t i = 1; i < sizeof(gOpName) / sizeof(gOpName[0]); ++i) {
+    if (strcmp(name, gOpName[i])) {
+      return i;
+    }
+  }
+  return OP_Unknown;
+}
 
 static void toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) {
   SWAP(dataBlockSchema->pSchema, pPlanNode->pSchema, SSchema*);
@@ -179,7 +188,7 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
       assert(false);
   }
   if (pPlanNode->pChildren != NULL && taosArrayGetSize(pPlanNode->pChildren) > 0) {
-    node->pChildren = taosArrayInit(4, POINTER_BYTES);
+    node->pChildren = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
     size_t size = taosArrayGetSize(pPlanNode->pChildren);
     for(int32_t i = 0; i < size; ++i) {
       SPhyNode* child = createPhyNode(pCxt, taosArrayGet(pPlanNode->pChildren, i));
@@ -215,3 +224,7 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD
   *pDag = context.pDag;
   return TSDB_CODE_SUCCESS;
 }
+
+int32_t setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SArray* eps) {
+  //todo
+}

source/libs/planner/src/planner.c

@@ -16,6 +16,10 @@
 #include "parser.h"
 #include "plannerInt.h"
 
+void qDestroySubplan(SSubplan* pSubplan) {
+  // todo
+}
+
 void qDestroyQueryDag(struct SQueryDag* pDag) {
   // todo
 }
@@ -42,6 +46,10 @@ int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet*
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SArray* eps) {
+  return setSubplanExecutionNode(subplan, templateId, eps);
+}
+
 int32_t qSubPlanToString(const SSubplan *subplan, char** str) {
   return subPlanToString(subplan, str);
 }

source/libs/planner/test/CMakeLists.txt

@@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 ADD_EXECUTABLE(plannerTest ${SOURCE_LIST})
 TARGET_LINK_LIBRARIES(
         plannerTest
-        PUBLIC os util common planner parser catalog transport gtest function query
+        PUBLIC os util common planner parser catalog transport gtest function qcom
 )
 
 TARGET_INCLUDE_DIRECTORIES(

source/libs/query/CMakeLists.txt → source/libs/qcom/CMakeLists.txt

 aux_source_directory(src QUERY_SRC)
-add_library(query ${QUERY_SRC})
+add_library(qcom ${QUERY_SRC})
 target_include_directories(
-    query
-    PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/query"
+    qcom
+    PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qcom"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 
 target_link_libraries(
-   query
-   PRIVATE os util common transport
+   qcom
+   PRIVATE os util transport
 )

source/libs/qcom/src/queryUtil.c

+#include "os.h"
+#include "taosmsg.h"
+
+#define VALIDNUMOFCOLS(x)  ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS)
+#define VALIDNUMOFTAGS(x)  ((x) >= 0 && (x) <= TSDB_MAX_TAGS)
+
+static struct SSchema _s = {
+    .colId = TSDB_TBNAME_COLUMN_INDEX,
+    .type  = TSDB_DATA_TYPE_BINARY,
+    .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE,
+    .name = "tbname",
+};
+
+SSchema* tGetTbnameColumnSchema() {
+  return &_s;
+}
+
+static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen) {
+  int32_t rowLen = 0;
+
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    // 1. valid types
+    if (!isValidDataType(pSchema[i].type)) {
+      return false;
+    }
+
+    // 2. valid length for each type
+    if (pSchema[i].type == TSDB_DATA_TYPE_BINARY) {
+      if (pSchema[i].bytes > TSDB_MAX_BINARY_LEN) {
+        return false;
+      }
+    } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) {
+      if (pSchema[i].bytes > TSDB_MAX_NCHAR_LEN) {
+        return false;
+      }
+    } else {
+      if (pSchema[i].bytes != tDataTypes[pSchema[i].type].bytes) {
+        return false;
+      }
+    }
+
+    // 3. valid column names
+    for (int32_t j = i + 1; j < numOfCols; ++j) {
+      if (strncasecmp(pSchema[i].name, pSchema[j].name, sizeof(pSchema[i].name) - 1) == 0) {
+        return false;
+      }
+    }
+
+    rowLen += pSchema[i].bytes;
+  }
+
+  return rowLen <= maxLen;
+}
+
+bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags) {
+  if (!VALIDNUMOFCOLS(numOfCols)) {
+    return false;
+  }
+
+  if (!VALIDNUMOFTAGS(numOfTags)) {
+    return false;
+  }
+
+  /* first column must be the timestamp, which is a primary key */
+  if (pSchema[0].type != TSDB_DATA_TYPE_TIMESTAMP) {
+    return false;
+  }
+
+  if (!doValidateSchema(pSchema, numOfCols, TSDB_MAX_BYTES_PER_ROW)) {
+    return false;
+  }
+
+  if (!doValidateSchema(&pSchema[numOfCols], numOfTags, TSDB_MAX_TAGS_LEN)) {
+    return false;
+  }
+
+  return true;
+}
\ No newline at end of file

source/libs/scheduler/CMakeLists.txt

@@ -9,5 +9,5 @@ target_include_directories(
 
 target_link_libraries(
         scheduler
-        PRIVATE os util planner common query
+        PRIVATE os util planner qcom common
 )
\ No newline at end of file

source/util/src/terror.c

@@ -130,6 +130,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_VALUE_OUT_OF_RANGE,       "Value out of range")
 TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_INPUT,            "Invalid tsc input")
 
 // mnode-common
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR,                "Mnode internal error")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY,                "Cluster not ready")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED,        "Message not processed")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_IN_PROGRESS,       "Message is progressing")

tests/CMakeLists.txt

-# generate debug version:
-# mkdir debug; cd debug; cmake -DCMAKE_BUILD_TYPE=Debug ..
-# generate release version:
-# mkdir release; cd release; cmake -DCMAKE_BUILD_TYPE=Release ..
-
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20)
-PROJECT(TDengine)
-
-SET(CMAKE_C_STANDARD 11)
-SET(CMAKE_VERBOSE_MAKEFILE ON)
-
-ADD_SUBDIRECTORY(examples/c)
+#ADD_SUBDIRECTORY(examples/c)
 ADD_SUBDIRECTORY(tsim)
-ADD_SUBDIRECTORY(test/c)
-ADD_SUBDIRECTORY(comparisonTest/tdengine)
+#ADD_SUBDIRECTORY(test/c)
+#ADD_SUBDIRECTORY(comparisonTest/tdengine)

tests/script/general/user/basic1.sim

@@ -5,7 +5,7 @@ sql connect
 
 print =============== show users
 sql show users
-if $rows != 3 then 
+if $rows != 1 then 
   return -1
 endi
 
@@ -21,7 +21,7 @@ sql_error drop account root
 print =============== create user1
 sql create user user1 PASS 'user1'
 sql show users
-if $rows != 4 then 
+if $rows != 2 then 
   return -1
 endi
 
@@ -33,7 +33,7 @@ print $data30 $data31 $data32
 print =============== create user2
 sql create user user2 PASS 'user2'
 sql show users
-if $rows != 5 then 
+if $rows != 3 then 
   return -1
 endi
 
@@ -46,7 +46,7 @@ print $data40 $data41 $data42
 print =============== drop user1
 sql drop user user1
 sql show users
-if $rows != 4 then 
+if $rows != 2 then 
   return -1
 endi
 
@@ -62,7 +62,7 @@ system sh/exec.sh -n dnode1 -s start
 
 print =============== show users
 sql show users
-if $rows != 4 then 
+if $rows != 2 then 
   return -1
 endi
 

tests/script/sh/deploy.sh

 #!/bin/bash
 
+set +e
+#set -x
+
 echo "Executing deploy.sh"
 
 if [ $# != 4 ]; then
@@ -50,12 +53,12 @@ else
 fi
 
 if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then
-  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3`
+  BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2,3`
 else
-  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2`
+  BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2`
 fi
 
-BUILD_DIR=$TAOS_DIR/$BIN_DIR/build
+BUILD_DIR=$TAOS_DIR/$BIN_DIR
 
 SIM_DIR=$TAOS_DIR/sim
 

tests/script/sh/exec.sh

@@ -8,6 +8,9 @@
   # exit 1
 # fi
 
+set +e
+#set -x
+
 UNAME_BIN=`which uname`
 OS_TYPE=`$UNAME_BIN`
 
@@ -62,16 +65,16 @@ else
 fi
 
 if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then
-  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3`
+  BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2,3`
 else
-  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2`
+  BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2`
 fi
 
-BUILD_DIR=$TAOS_DIR/$BIN_DIR/build
+BUILD_DIR=$TAOS_DIR/$BIN_DIR
 
 SIM_DIR=$TAOS_DIR/sim
 NODE_DIR=$SIM_DIR/$NODE_NAME
-EXE_DIR=$BUILD_DIR/bin
+EXE_DIR=$BUILD_DIR/source/dnode/mgmt/daemon
 CFG_DIR=$NODE_DIR/cfg
 LOG_DIR=$NODE_DIR/log
 DATA_DIR=$NODE_DIR/data

tests/script/test.sh

@@ -22,9 +22,6 @@ do
     f)
       FILE_NAME=$OPTARG
       ;;
-    a)
-      ASYNC=1
-      ;;
     v)
       VALGRIND=1
       ;;
@@ -60,32 +57,22 @@ else
 fi
 
 if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then
-  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3`
+  BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2,3`
 else
-  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2`
+  BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2`
 fi
 
-BUILD_DIR=$TOP_DIR/$BIN_DIR/build
+BUILD_DIR=$TOP_DIR/$BIN_DIR
 
 SIM_DIR=$TOP_DIR/sim
 
-if [ $ASYNC -eq 0 ]; then
-  PROGRAM=$BUILD_DIR/bin/tsim
-else
-  PROGRAM="$BUILD_DIR/bin/tsim -a"
-fi
-
+PROGRAM=$BUILD_DIR/tests/tsim/tsim
 
 PRG_DIR=$SIM_DIR/tsim
 CFG_DIR=$PRG_DIR/cfg
 LOG_DIR=$PRG_DIR/log
 DATA_DIR=$PRG_DIR/data
 
-
-ARBITRATOR_PRG_DIR=$SIM_DIR/arbitrator
-ARBITRATOR_LOG_DIR=$ARBITRATOR_PRG_DIR/log
-
-
 chmod -R 777 $PRG_DIR
 echo "------------------------------------------------------------------------"
 echo "Start TDengine Testing Case ..."
@@ -96,12 +83,10 @@ echo "CFG_DIR  : $CFG_DIR"
 
 rm -rf $LOG_DIR
 rm -rf $CFG_DIR
-rm -rf $ARBITRATOR_LOG_DIR
 
 mkdir -p $PRG_DIR
 mkdir -p $LOG_DIR
 mkdir -p $CFG_DIR
-mkdir -p $ARBITRATOR_LOG_DIR
 
 TAOS_CFG=$PRG_DIR/cfg/taos.cfg
 touch -f $TAOS_CFG
@@ -115,7 +100,7 @@ echo "secondEp           ${HOSTNAME}:7200"        >> $TAOS_CFG
 echo "serverPort         7100"                    >> $TAOS_CFG
 echo "dataDir            $DATA_DIR"               >> $TAOS_CFG
 echo "logDir             $LOG_DIR"                >> $TAOS_CFG
-echo "scriptDir          ${CODE_DIR}/../script"   >> $TAOS_CFG
+echo "scriptDir          ${CODE_DIR}"             >> $TAOS_CFG
 echo "numOfLogLines      100000000"               >> $TAOS_CFG
 echo "rpcDebugFlag       143"                     >> $TAOS_CFG
 echo "tmrDebugFlag       131"                     >> $TAOS_CFG
@@ -141,7 +126,6 @@ if [ -n "$FILE_NAME" ]; then
   else
     echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f $FILE_NAME
     $PROGRAM -c $CFG_DIR -f $FILE_NAME
-#    valgrind --tool=memcheck --leak-check=full --show-reachable=no  --track-origins=yes --show-leak-kinds=all  -v  --workaround-gcc296-bugs=yes  --log-file=${CODE_DIR}/../script/valgrind.log $PROGRAM -c $CFG_DIR -f $FILE_NAME
   fi
 else
   echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f basicSuite.sim

tests/tsim/CMakeLists.txt

-PROJECT(TDengine)
-
-INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
-INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
-INCLUDE_DIRECTORIES(inc)
-
-AUX_SOURCE_DIRECTORY(src SRC)
-ADD_EXECUTABLE(tsim ${SRC})
-TARGET_LINK_LIBRARIES(tsim taos_static trpc tutil pthread cJson)
+aux_source_directory(src TSIM_SRC)
+add_executable(tsim ${TSIM_SRC})
+target_link_libraries(
+    tsim
+    PUBLIC taos
+    PUBLIC util
+    PUBLIC common
+    PUBLIC os
+    PUBLIC cjson
+)
+target_include_directories(
+    tsim
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+)

tests/tsim/inc/sim.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef __SIM_H__
-#define __SIM_H__
+#ifndef _TD_SIM_H_
+#define _TD_SIM_H_
 
 #include <semaphore.h>
 #include <stdbool.h>
@@ -102,18 +102,18 @@ typedef struct _cmd_t {
   int16_t cmdno;
   int16_t nlen;
   char    name[MAX_SIM_CMD_NAME_LEN];
-  bool  (*parseCmd)(char *, struct _cmd_t *, int32_t);
-  bool  (*executeCmd)(struct _script_t *script, char *option);
+  bool (*parseCmd)(char *, struct _cmd_t *, int32_t);
+  bool (*executeCmd)(struct _script_t *script, char *option);
   struct _cmd_t *next;
 } SCommand;
 
 typedef struct {
   int16_t cmdno;
-  int16_t jump;        // jump position
-  int16_t errorJump;   // sql jump flag, while '-x' exist in sql cmd, this flag
-                       // will be SQL_JUMP_TRUE, otherwise is SQL_JUMP_FALSE */
-  int16_t lineNum;     // correspodning line number in original file
-  int32_t optionOffset;// relative option offset
+  int16_t jump;          // jump position
+  int16_t errorJump;     // sql jump flag, while '-x' exist in sql cmd, this flag
+                         // will be SQL_JUMP_TRUE, otherwise is SQL_JUMP_FALSE */
+  int16_t lineNum;       // correspodning line number in original file
+  int32_t optionOffset;  // relative option offset
 } SCmdLine;
 
 typedef struct _var_t {
@@ -123,24 +123,24 @@ typedef struct _var_t {
 } SVariable;
 
 typedef struct _script_t {
-  int32_t   type;
-  bool      killed;
-  void *    taos;
-  char      rows[12];                                                         // number of rows data retrieved
-  char      data[MAX_QUERY_ROW_NUM][MAX_QUERY_COL_NUM][MAX_QUERY_VALUE_LEN];  // query results
-  char      system_exit_code[12];
-  char      system_ret_content[MAX_SYSTEM_RESULT_LEN];
-  int32_t   varLen;
-  int32_t   linePos;     // current cmd position
-  int32_t   numOfLines;  // number of lines in the script
-  int32_t   bgScriptLen;
-  char      fileName[MAX_FILE_NAME_LEN];  // script file name
-  char      error[MAX_ERROR_LEN];
-  char *    optionBuffer;
-  SCmdLine *lines;  // command list
-  SVariable variables[MAX_VAR_LEN];
-  pthread_t bgPid;
-  char      auth[128];
+  int32_t           type;
+  bool              killed;
+  void             *taos;
+  char              rows[12];                                                         // number of rows data retrieved
+  char              data[MAX_QUERY_ROW_NUM][MAX_QUERY_COL_NUM][MAX_QUERY_VALUE_LEN];  // query results
+  char              system_exit_code[12];
+  char              system_ret_content[MAX_SYSTEM_RESULT_LEN];
+  int32_t           varLen;
+  int32_t           linePos;     // current cmd position
+  int32_t           numOfLines;  // number of lines in the script
+  int32_t           bgScriptLen;
+  char              fileName[MAX_FILE_NAME_LEN];  // script file name
+  char              error[MAX_ERROR_LEN];
+  char             *optionBuffer;
+  SCmdLine         *lines;  // command list
+  SVariable         variables[MAX_VAR_LEN];
+  pthread_t         bgPid;
+  char              auth[128];
   struct _script_t *bgScripts[MAX_BACKGROUND_SCRIPT_NUM];
 } SScript;
 
@@ -150,16 +150,15 @@ extern int32_t  simScriptPos;
 extern int32_t  simScriptSucced;
 extern int32_t  simDebugFlag;
 extern char     tsScriptDir[];
-extern bool     simAsyncQuery;
 extern bool     abortExecution;
 
 SScript *simParseScript(char *fileName);
 SScript *simProcessCallOver(SScript *script);
-void *   simExecuteScript(void *script);
+void    *simExecuteScript(void *script);
 void     simInitsimCmdList();
 bool     simSystemInit();
 void     simSystemCleanUp();
-char *   simGetVariable(SScript *script, char *varName, int32_t varLen);
+char    *simGetVariable(SScript *script, char *varName, int32_t varLen);
 bool     simExecuteExpCmd(SScript *script, char *option);
 bool     simExecuteTestCmd(SScript *script, char *option);
 bool     simExecuteGotoCmd(SScript *script, char *option);
@@ -178,4 +177,4 @@ bool     simExecuteLineInsertCmd(SScript *script, char *option);
 bool     simExecuteLineInsertErrorCmd(SScript *script, char *option);
 void     simVisuallizeOption(SScript *script, char *src, char *dst);
 
-#endif
+#endif /*_TD_SIM_H_*/
\ No newline at end of file

tests/tsim/inc/simParse.h

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef __SIM_PARSE_H__
-#define __SIM_PARSE_H__
+#ifndef _TD_SIM_PARSE_H_
+#define _TD_SIM_PARSE_H_
 
 #define MAX_NUM_CMD 64
 #define MAX_NUM_LABLES 100
@@ -40,10 +40,10 @@ typedef struct {
 
 /* block definition */
 typedef struct {
-  char     top;                  /* the number of blocks stacked */
-  char     type[MAX_NUM_BLOCK];  /* the block type */
-  int16_t *pos[MAX_NUM_BLOCK];   /* position of the jump for if/elif/case */
-  int16_t  back[MAX_NUM_BLOCK];  /* go back, endw and continue */
+  char     top;                 /* the number of blocks stacked */
+  char     type[MAX_NUM_BLOCK]; /* the block type */
+  int16_t *pos[MAX_NUM_BLOCK];  /* position of the jump for if/elif/case */
+  int16_t  back[MAX_NUM_BLOCK]; /* go back, endw and continue */
   char     numJump[MAX_NUM_BLOCK];
   int16_t *jump[MAX_NUM_BLOCK][MAX_NUM_JUMP]; /* break or elif */
   char     sexp[MAX_NUM_BLOCK][40];           /*switch expression */
@@ -52,4 +52,4 @@ typedef struct {
 
 bool simParseExpression(char *token, int32_t lineNum);
 
-#endif
+#endif /*_TD_SIM_PARSE_H_*/
\ No newline at end of file

tests/tsim/src/simExe.c

@@ -14,18 +14,18 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "../../../include/client/taos.h"
 #include "cJSON.h"
 #include "os.h"
 #include "sim.h"
+#include "taos.h"
 #include "taoserror.h"
 #include "tglobal.h"
+#include "ttypes.h"
 #include "tutil.h"
-#undef TAOS_MEM_CHECK
 
 void simLogSql(char *sql, bool useSharp) {
   static FILE *fp = NULL;
-  char filename[256];
+  char         filename[256];
   sprintf(filename, "%s/sim.sql", tsScriptDir);
   if (fp == NULL) {
     fp = fopen(filename, "w");
@@ -74,7 +74,7 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) {
         return "null";
       }
 
-      char *  keyName;
+      char   *keyName;
       int32_t keyLen;
       paGetToken(varName + 6, &keyName, &keyLen);
 
@@ -91,7 +91,7 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) {
         return "null";
       }
 
-      char *  keyName;
+      char   *keyName;
       int32_t keyLen;
       paGetToken(varName + 7, &keyName, &keyLen);
 
@@ -144,7 +144,7 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) {
 }
 
 int32_t simExecuteExpression(SScript *script, char *exp) {
-  char *  op1, *op2, *var1, *var2, *var3, *rest;
+  char   *op1, *op2, *var1, *var2, *var3, *rest;
   int32_t op1Len, op2Len, var1Len, var2Len, var3Len, val0, val1;
   char    t0[1024], t1[1024], t2[1024], t3[2048];
   int32_t result;
@@ -302,10 +302,10 @@ bool simExecuteRunBackCmd(SScript *script, char *option) {
 }
 
 void simReplaceShToBat(char *dst) {
-  char* sh = strstr(dst, ".sh");
+  char *sh = strstr(dst, ".sh");
   if (sh != NULL) {
     int32_t dstLen = (int32_t)strlen(dst);
-    char *end = dst + dstLen;
+    char   *end = dst + dstLen;
     *(end + 1) = 0;
 
     for (char *p = end; p >= sh; p--) {
@@ -436,7 +436,7 @@ bool simExecuteReturnCmd(SScript *script, char *option) {
 }
 
 void simVisuallizeOption(SScript *script, char *src, char *dst) {
-  char *  var, *token, *value;
+  char   *var, *token, *value;
   int32_t dstLen, srcLen, tokenLen;
 
   dst[0] = 0, dstLen = 0;
@@ -466,10 +466,6 @@ void simVisuallizeOption(SScript *script, char *src, char *dst) {
   strcpy(dst + dstLen, src);
 }
 
-void simCloseRestFulConnect(SScript *script) { 
-  memset(script->auth, 0, sizeof(script->auth));
-}
-
 void simCloseNativeConnect(SScript *script) {
   if (script->taos == NULL) return;
 
@@ -479,168 +475,7 @@ void simCloseNativeConnect(SScript *script) {
   script->taos = NULL;
 }
 
-void simCloseTaosdConnect(SScript *script) {
-  if (simAsyncQuery) {
-    simCloseRestFulConnect(script);
-  } else {
-    simCloseNativeConnect(script);
-  }
-}
-//  {"status":"succ","code":0,"desc":"/KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04"}
-//  {"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}
-//  {"status":"succ","head":["ts","i"],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10]],"rows":10}
-int32_t simParseHttpCommandResult(SScript *script, char *command) {
-  cJSON* root = cJSON_Parse(command);
-  if (root == NULL) {
-    simError("script:%s, failed to parse json, response:%s", script->fileName, command);
-    return -1;
-  }
-
-  cJSON *status = cJSON_GetObjectItem(root, "status");
-  if (status == NULL) {
-    simError("script:%s, failed to parse json, status is null, response:%s", script->fileName, command);
-    cJSON_Delete(root);
-    return -1;
-  }
-
-  if (status->valuestring == NULL || strlen(status->valuestring) == 0) {
-    simError("script:%s, failed to parse json, status value is null, response:%s", script->fileName, command);
-    cJSON_Delete(root);
-    return -1;
-  }
-
-  if (strcmp(status->valuestring, "succ") != 0) {
-    cJSON *code = cJSON_GetObjectItem(root, "code");
-    if (code == NULL) {
-      simError("script:%s, failed to parse json, code is null, response:%s", script->fileName, command);
-      cJSON_Delete(root);
-      return -1;
-    }
-    int32_t retcode = (int32_t)code->valueint;
-    if (retcode != 1017) {
-      simError("script:%s, json:status:%s not equal to succ, response:%s", script->fileName, status->valuestring,
-               command);
-      cJSON_Delete(root);
-      return retcode;
-    } else {
-      simDebug("script:%s, json:status:%s not equal to succ, but code is %d, response:%s", script->fileName,
-               status->valuestring, retcode, command);
-      cJSON_Delete(root);
-      return 0;
-    }
-  }
-
-  cJSON *desc = cJSON_GetObjectItem(root, "desc");
-  if (desc != NULL) {
-    if (desc->valuestring == NULL || strlen(desc->valuestring) == 0) {
-      simError("script:%s, failed to parse json, desc value is null, response:%s", script->fileName, command);
-      cJSON_Delete(root);
-      return -1;
-    }
-    strcpy(script->auth, desc->valuestring);
-    cJSON_Delete(root);
-    return 0;
-  }
-
-  cJSON *data = cJSON_GetObjectItem(root, "data");
-  if (data == NULL) {
-    simError("script:%s, failed to parse json, data is null, response:%s", script->fileName, command);
-    cJSON_Delete(root);
-    return -1;
-  }
-
-  int32_t rowsize = cJSON_GetArraySize(data);
-  if (rowsize < 0) {
-    simError("script:%s, failed to parse json:data, data size %d, response:%s", script->fileName, rowsize, command);
-    cJSON_Delete(root);
-    return -1;
-  }
-
-  int32_t rowIndex = 0;
-  sprintf(script->rows, "%d", rowsize);
-  for (int32_t r = 0; r < rowsize; ++r) {
-    cJSON *row = cJSON_GetArrayItem(data, r);
-    if (row == NULL) continue;
-    if (rowIndex++ >= 10) break;
-
-    int32_t colsize = cJSON_GetArraySize(row);
-    if (colsize < 0) {
-      break;
-    }
-
-    colsize = MIN(10, colsize);
-    for (int32_t c = 0; c < colsize; ++c) {
-      cJSON *col = cJSON_GetArrayItem(row, c);
-      if (col->valuestring != NULL) {
-        strcpy(script->data[r][c], col->valuestring);
-      } else {
-        if (col->numberstring[0] == 0) {
-          strcpy(script->data[r][c], "null");
-        } else {
-          strcpy(script->data[r][c], col->numberstring);
-        }
-      }
-    }
-  }
-
-  return 0;
-}
-
-int32_t simExecuteRestFulCommand(SScript *script, char *command) {
-  char buf[5000] = {0};
-  sprintf(buf, "%s 2>/dev/null", command);
-
-  FILE *fp = popen(buf, "r");
-  if (fp == NULL) {
-    simError("failed to execute %s", buf);
-    return -1;
-  }
-
-  int32_t mallocSize = 2000;
-  int32_t alreadyReadSize = 0;
-  char *  content = malloc(mallocSize);
-
-  while (!feof(fp)) {
-    int32_t availSize = mallocSize - alreadyReadSize;
-    int32_t len = (int32_t)fread(content + alreadyReadSize, 1, availSize, fp);
-    if (len >= availSize) {
-      alreadyReadSize += len;
-      mallocSize *= 2;
-      content = realloc(content, mallocSize);
-    }
-  }
-
-  pclose(fp);
-
-  return simParseHttpCommandResult(script, content);
-}
-
-bool simCreateRestFulConnect(SScript *script, char *user, char *pass) {
-  char command[4096];
-  sprintf(command, "curl 127.0.0.1:6041/rest/login/%s/%s", user, pass);
-
-  bool success = false;
-  for (int32_t attempt = 0; attempt < 10; ++attempt) {
-    success = simExecuteRestFulCommand(script, command) == 0;
-    if (!success) {
-      simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL),
-               attempt);
-      taosMsleep(1000);
-    } else {
-      simDebug("script:%s, user:%s connect taosd successed, attempt:%d", script->fileName, user, attempt);
-      break;
-    }
-  }
-
-  if (!success) {
-    sprintf(script->error, "lineNum:%d. connect taosd failed:%s", script->lines[script->linePos].lineNum,
-            taos_errstr(NULL));
-    return false;
-  }
-
-  simDebug("script:%s, connect taosd successed, auth:%p", script->fileName, script->auth);
-  return true;
-}
+void simCloseTaosdConnect(SScript *script) { simCloseNativeConnect(script); }
 
 bool simCreateNativeConnect(SScript *script, char *user, char *pass) {
   simCloseTaosdConnect(script);
@@ -651,7 +486,7 @@ bool simCreateNativeConnect(SScript *script, char *user, char *pass) {
       return false;
     }
 
-    taos = taos_connect(NULL, user, pass, NULL, tsDnodeShellPort);
+    taos = taos_connect(NULL, user, pass, NULL, 0);
     if (taos == NULL) {
       simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL),
                attempt);
@@ -675,8 +510,8 @@ bool simCreateNativeConnect(SScript *script, char *user, char *pass) {
 }
 
 bool simCreateTaosdConnect(SScript *script, char *rest) {
-  char *  user = TSDB_DEFAULT_USER;
-  char *  token;
+  char   *user = TSDB_DEFAULT_USER;
+  char   *token;
   int32_t tokenLen;
   rest = paGetToken(rest, &token, &tokenLen);
   rest = paGetToken(rest, &token, &tokenLen);
@@ -684,18 +519,14 @@ bool simCreateTaosdConnect(SScript *script, char *rest) {
     user = token;
   }
 
-  if (simAsyncQuery) {
-    return simCreateRestFulConnect(script, user, TSDB_DEFAULT_PASS);
-  } else {
-    return simCreateNativeConnect(script, user, TSDB_DEFAULT_PASS);
-  }
+  return simCreateNativeConnect(script, user, TSDB_DEFAULT_PASS);
 }
 
 bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
   char       timeStr[30] = {0};
   time_t     tt;
   struct tm *tp;
-  SCmdLine * line = &script->lines[script->linePos];
+  SCmdLine  *line = &script->lines[script->linePos];
   int32_t    ret = -1;
 
   TAOS_RES *pSql = NULL;
@@ -710,7 +541,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
     pSql = taos_query(script->taos, rest);
     ret = taos_errno(pSql);
 
-    if (ret == TSDB_CODE_MND_TABLE_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) {
+    if (ret == TSDB_CODE_MND_STB_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) {
       simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF,
                tstrerror(ret));
       ret = 0;
@@ -756,7 +587,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
     while ((row = taos_fetch_row(pSql))) {
       if (numOfRows < MAX_QUERY_ROW_NUM) {
         TAOS_FIELD *fields = taos_fetch_fields(pSql);
-        int32_t *   length = taos_fetch_lengths(pSql);
+        int32_t    *length = taos_fetch_lengths(pSql);
 
         for (int32_t i = 0; i < num_fields; i++) {
           char *value = NULL;
@@ -780,7 +611,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
               sprintf(value, "%d", *((int8_t *)row[i]));
               break;
             case TSDB_DATA_TYPE_UTINYINT:
-              sprintf(value, "%u", *((uint8_t*)row[i]));
+              sprintf(value, "%u", *((uint8_t *)row[i]));
               break;
             case TSDB_DATA_TYPE_SMALLINT:
               sprintf(value, "%d", *((int16_t *)row[i]));
@@ -846,7 +677,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
               } else if (precision == TSDB_TIME_PRECISION_MICRO) {
                 sprintf(value, "%s.%06d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000000));
               } else {
-                sprintf(value, "%s.%09d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000000000));                
+                sprintf(value, "%s.%09d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000000000));
               }
 
               break;
@@ -877,43 +708,8 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) {
   return true;
 }
 
-bool simExecuteRestFulSqlCommand(SScript *script, char *rest) {
-  SCmdLine *line = &script->lines[script->linePos];
-  char command[4096];
-  sprintf(command, "curl -H 'Authorization: Taosd %s' -d \"%s\" 127.0.0.1:6041/rest/sql", script->auth, rest);
-
-  int32_t ret = -1;
-  for (int32_t attempt = 0; attempt < 10; ++attempt) {
-    ret = simExecuteRestFulCommand(script, command);
-    if (ret == TSDB_CODE_MND_TABLE_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) {
-      simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF,
-               tstrerror(ret));
-      ret = 0;
-      break;
-    } else if (ret != 0) {
-      simDebug("script:%s, taos:%p, %s failed, ret:%d", script->fileName, script->taos, rest, ret);
-
-      if (line->errorJump == SQL_JUMP_TRUE) {
-        script->linePos = line->jump;
-        return true;
-      }
-      taosMsleep(1000);
-    } else {
-      break;
-    }
-  }
-
-  if (ret) {
-    sprintf(script->error, "lineNum:%d. sql:%s failed, ret:%d", line->lineNum, rest, ret);
-    return false;
-  }
-
-  script->linePos++;
-  return true;
-}
-
 bool simExecuteSqlImpCmd(SScript *script, char *rest, bool isSlow) {
-  char buf[3000];
+  char      buf[3000];
   SCmdLine *line = &script->lines[script->linePos];
 
   simVisuallizeOption(script, rest, buf);
@@ -935,7 +731,7 @@ bool simExecuteSqlImpCmd(SScript *script, char *rest, bool isSlow) {
     return true;
   }
 
-  if ((!simAsyncQuery && script->taos == NULL) || (simAsyncQuery && script->auth[0] == 0)) {
+  if (script->taos == NULL) {
     if (!simCreateTaosdConnect(script, "connect root")) {
       if (line->errorJump == SQL_JUMP_TRUE) {
         script->linePos = line->jump;
@@ -951,11 +747,7 @@ bool simExecuteSqlImpCmd(SScript *script, char *rest, bool isSlow) {
     return true;
   }
 
-  if (simAsyncQuery) {
-    return simExecuteRestFulSqlCommand(script, rest);
-  } else {
-    return simExecuteNativeSqlCommand(script, rest, isSlow);
-  }
+  return simExecuteNativeSqlCommand(script, rest, isSlow);
 }
 
 bool simExecuteSqlCmd(SScript *script, char *rest) {
@@ -1010,7 +802,7 @@ bool simExecuteRestfulCmd(SScript *script, char *rest) {
 }
 
 bool simExecuteSqlErrorCmd(SScript *script, char *rest) {
-  char buf[3000];
+  char      buf[3000];
   SCmdLine *line = &script->lines[script->linePos];
 
   simVisuallizeOption(script, rest, buf);
@@ -1032,7 +824,7 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) {
     return true;
   }
 
-  if ((!simAsyncQuery && script->taos == NULL) || (simAsyncQuery && script->auth[0] == 0)) {
+  if (script->taos == NULL) {
     if (!simCreateTaosdConnect(script, "connect root")) {
       if (line->errorJump == SQL_JUMP_TRUE) {
         script->linePos = line->jump;
@@ -1048,17 +840,9 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) {
     return true;
   }
 
-  int32_t   ret;
-  TAOS_RES *pSql = NULL;
-  if (simAsyncQuery) {
-    char command[4096];
-    sprintf(command, "curl -H 'Authorization: Taosd %s' -d '%s' 127.0.0.1:6041/rest/sql", script->auth, rest);
-    ret = simExecuteRestFulCommand(script, command);
-  } else {
-    pSql = taos_query(script->taos, rest);
-    ret = taos_errno(pSql);
-    taos_free_result(pSql);
-  }
+  TAOS_RES *pSql = pSql = taos_query(script->taos, rest);
+  int32_t   ret = taos_errno(pSql);
+  taos_free_result(pSql);
 
   if (ret != TSDB_CODE_SUCCESS) {
     simDebug("script:%s, taos:%p, %s execute, expect failed, so success, ret:%d:%s", script->fileName, script->taos,
@@ -1083,15 +867,19 @@ bool simExecuteLineInsertCmd(SScript *script, char *rest) {
 
   simInfo("script:%s, %s", script->fileName, rest);
   simLogSql(buf, true);
-  char *  lines[] = {rest};
+  char *lines[] = {rest};
+#if 0
   int32_t ret = taos_insert_lines(script->taos, lines, 1);
+#else
+  int32_t ret = 0;
+#endif
   if (ret == TSDB_CODE_SUCCESS) {
     simDebug("script:%s, taos:%p, %s executed. success.", script->fileName, script->taos, rest);
     script->linePos++;
     return true;
   } else {
-    sprintf(script->error, "lineNum: %d. line: %s failed, ret:%d:%s", line->lineNum, rest,
-            ret & 0XFFFF, tstrerror(ret));
+    sprintf(script->error, "lineNum: %d. line: %s failed, ret:%d:%s", line->lineNum, rest, ret & 0XFFFF,
+            tstrerror(ret));
     return false;
   }
 }
@@ -1106,15 +894,20 @@ bool simExecuteLineInsertErrorCmd(SScript *script, char *rest) {
 
   simInfo("script:%s, %s", script->fileName, rest);
   simLogSql(buf, true);
-  char *  lines[] = {rest};
+  char *lines[] = {rest};
+#if 0
   int32_t ret = taos_insert_lines(script->taos, lines, 1);
+#else
+  int32_t ret = 0;
+#endif
   if (ret == TSDB_CODE_SUCCESS) {
-    sprintf(script->error, "script:%s, taos:%p, %s executed. expect failed, but success.", script->fileName, script->taos, rest);
+    sprintf(script->error, "script:%s, taos:%p, %s executed. expect failed, but success.", script->fileName,
+            script->taos, rest);
     script->linePos++;
     return false;
   } else {
-    simDebug("lineNum: %d. line: %s failed, ret:%d:%s. Expect failed, so success", line->lineNum, rest,
-            ret & 0XFFFF, tstrerror(ret));
+    simDebug("lineNum: %d. line: %s failed, ret:%d:%s. Expect failed, so success", line->lineNum, rest, ret & 0XFFFF,
+             tstrerror(ret));
     return true;
   }
 }

tests/tsim/src/simMain.c

@@ -15,19 +15,17 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "tglobal.h"
 #include "sim.h"
-#undef TAOS_MEM_CHECK
+#include "tglobal.h"
 
-bool simAsyncQuery = false;
 bool simExecSuccess = false;
 bool abortExecution = false;
 
 void simHandleSignal(int32_t signo, void *sigInfo, void *context) {
   simSystemCleanUp();
   abortExecution = true;
-//  runningScript->killed = true;
-//  exit(1);
+  //  runningScript->killed = true;
+  //  exit(1);
 }
 
 int32_t main(int32_t argc, char *argv[]) {
@@ -38,8 +36,6 @@ int32_t main(int32_t argc, char *argv[]) {
       tstrncpy(configDir, argv[++i], 128);
     } else if (strcmp(argv[i], "-f") == 0 && i < argc - 1) {
       strcpy(scriptFile, argv[++i]);
-    } else if (strcmp(argv[i], "-a") == 0) {
-      simAsyncQuery = true;
     } else {
       printf("usage: %s [options] \n", argv[0]);
       printf("       [-c config]: config directory, default is: %s\n", configDir);

tests/tsim/src/simSystem.c

@@ -14,15 +14,15 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "../../../include/client/taos.h"
 #include "os.h"
 #include "sim.h"
+#include "taos.h"
 #include "taoserror.h"
 #include "tglobal.h"
-#include "tsocket.h"
 #include "ttimer.h"
 #include "tutil.h"
-#undef TAOS_MEM_CHECK
+#include "tglobal.h"
+#include "tconfig.h"
 
 SScript *simScriptList[MAX_MAIN_SCRIPT_NUM];
 SCommand simCmdList[SIM_CMD_END];
@@ -81,10 +81,11 @@ char *simParseHostName(char *varName) {
 }
 
 bool simSystemInit() {
-  if (taos_init()) {
-    return false;
-  }
   taosGetFqdn(simHostName);
+
+  taosInitGlobalCfg();
+  taosReadCfgFromFile();
+
   simInitsimCmdList();
   memset(simScriptList, 0, sizeof(SScript *) * MAX_MAIN_SCRIPT_NUM);
   return true;
@@ -171,7 +172,7 @@ void *simExecuteScript(void *inputScript) {
       }
     } else {
       SCmdLine *line = &script->lines[script->linePos];
-      char *    option = script->optionBuffer + line->optionOffset;
+      char     *option = script->optionBuffer + line->optionOffset;
       simDebug("script:%s, line:%d with option \"%s\"", script->fileName, line->lineNum, option);
 
       SCommand *cmd = &simCmdList[line->cmdno];

tools/CMakeLists.txt

-#add_subdirectory(shell)
+add_subdirectory(shell)
\ No newline at end of file

tools/shell/CMakeLists.txt

 aux_source_directory(src SHELL_SRC)
-list(REMOVE_ITEM SHELL_SRC ./src/shellWindows.c)
-list(REMOVE_ITEM SHELL_SRC ./src/shellDarwin.c)
+list(REMOVE_ITEM SHELL_SRC src/shellWindows.c)
+list(REMOVE_ITEM SHELL_SRC src/shellDarwin.c)
 
 add_executable(shell ${SHELL_SRC})
 target_link_libraries(
     shell
     PUBLIC taos
     PUBLIC util
+    PUBLIC common
     PUBLIC os
 )
+target_include_directories(
+    shell
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+)
 
 SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos)

tools/shell/inc/shell.h

@@ -13,13 +13,13 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef __SHELL__
-#define __SHELL__
+#ifndef _TD_SHELL_H_
+#define _TD_SHELL_H_
 
-#include "../../../../include/client/taos.h"
-#include "stdbool.h"
+#include "os.h"
+
+#include "taos.h"
 #include "taosdef.h"
-#include "tsclient.h"
 
 #define MAX_USERNAME_SIZE      64
 #define MAX_DBNAME_SIZE        64