Merge branch '3.0' into catalog_dev

.gitignore

@@ -12,6 +12,7 @@ debug/
 release/
 target/
 debs/
+deps/
 rpms/
 mac/
 *.pyc
@@ -100,4 +101,4 @@ TAGS
 
 contrib/*
 !contrib/CMakeLists.txt
-!contrib/test
+!contrib/test
\ No newline at end of file

CMakeLists.txt

@@ -10,7 +10,7 @@ set(CMAKE_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/cmake")
 set(CMAKE_CONTRIB_DIR "${CMAKE_SOURCE_DIR}/contrib")
 include(${CMAKE_SUPPORT_DIR}/cmake.options)
 
-SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma")
+SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma -g3")
 
 # contrib
 add_subdirectory(contrib)
@@ -20,9 +20,13 @@ add_library(api INTERFACE)
 target_include_directories(api INTERFACE "include/client")
 
 # src
+if(${BUILD_TEST})
+    include(CTest)
+    enable_testing()
+endif(${BUILD_TEST})
 add_subdirectory(source)
 
 # docs 
 add_subdirectory(docs)
 
-# tests (TODO)
+# tests (TODO)
\ No newline at end of file

include/client/taos.h

@@ -73,10 +73,12 @@ typedef struct taosField {
   #define DLL_EXPORT 
 #endif
 
-DLL_EXPORT int   taos_init();
+typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code);
+
 DLL_EXPORT void  taos_cleanup(void);
 DLL_EXPORT int   taos_options(TSDB_OPTION option, const void *arg, ...);
 DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
+DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port);
 DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port);
 DLL_EXPORT void  taos_close(TAOS *taos);
 
@@ -154,14 +156,14 @@ DLL_EXPORT int* taos_fetch_lengths(TAOS_RES *res);
 // TAOS_RES   *taos_list_dbs(TAOS *mysql, const char *wild);
 
 // TODO: the return value should be `const`
-DLL_EXPORT char *taos_get_server_info(TAOS *taos);
-DLL_EXPORT char *taos_get_client_info();
-DLL_EXPORT char *taos_errstr(TAOS_RES *tres);
+DLL_EXPORT const char *taos_get_server_info(TAOS *taos);
+DLL_EXPORT const char *taos_get_client_info();
+DLL_EXPORT const char *taos_errstr(TAOS_RES *tres);
 
 DLL_EXPORT int taos_errno(TAOS_RES *tres);
 
-DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, int code), void *param);
-DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, int numOfRows), void *param);
+DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param);
+DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param);
 
 typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code);
 DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval);

include/common/taosmsg.h

@@ -74,9 +74,9 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_COMPACT_DB, "compact-db" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_FUNCTION, "create-function" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RETRIEVE_FUNCTION, "retrieve-function" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_FUNCTION, "drop-function" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STABLE, "create-stable" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STABLE, "alter-stable" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STABLE, "drop-stable" )	
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB, "create-stb" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB, "alter-stb" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB, "drop-stb" )	
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_VGROUP_LIST, "vgroup-list" )
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_QUERY, "kill-query" )	
 TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_STREAM, "kill-stream" )	
@@ -94,9 +94,9 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "nettest" )
 // message from vnode to dnode
 
 // message from mnode to vnode
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STABLE_IN, "create-stable" )
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STABLE_IN, "alter-stable" )	
-TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STABLE_IN, "drop-stable" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB_IN, "create-stb-in" )
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB_IN, "alter-stb-in" )	
+TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB_IN, "drop-stb-in" )
 // message from mnode to mnode
 // message from mnode to qnode
 // message from mnode to dnode
@@ -159,7 +159,7 @@ typedef enum _mgmt_table {
   TSDB_MGMT_TABLE_DNODE,
   TSDB_MGMT_TABLE_MNODE,
   TSDB_MGMT_TABLE_VGROUP,
-  TSDB_MGMT_TABLE_STABLE,
+  TSDB_MGMT_TABLE_STB,
   TSDB_MGMT_TABLE_MODULE,
   TSDB_MGMT_TABLE_QUERIES,
   TSDB_MGMT_TABLE_STREAMS,
@@ -299,7 +299,7 @@ typedef struct {
   uint64_t superTableUid;
   uint64_t createdTime;
   char     tableFname[TSDB_TABLE_FNAME_LEN];
-  char     stableFname[TSDB_TABLE_FNAME_LEN];
+  char     stbFname[TSDB_TABLE_FNAME_LEN];
   char     data[];
 } SMDCreateTableMsg;
 
@@ -316,16 +316,23 @@ typedef struct {
 } SCreateTableMsg;
 
 typedef struct {
-  int32_t numOfTables;
-  int32_t contLen;
-} SCMCreateTableMsg;
+  char    name[TSDB_TABLE_FNAME_LEN];
+  int8_t  igExists;
+  int32_t numOfTags;
+  int32_t numOfColumns;
+  SSchema pSchema[];
+} SCreateStbMsg;
 
 typedef struct {
   char   name[TSDB_TABLE_FNAME_LEN];
-  // if user specify DROP STABLE, this flag will be set. And an error will be returned if it is not a super table
-  int8_t supertable;
   int8_t igNotExists;
-} SCMDropTableMsg;
+} SDropStbMsg;
+
+typedef struct {
+  char    name[TSDB_TABLE_FNAME_LEN];
+  int8_t  alterType;
+  SSchema schema;
+} SAlterStbMsg;
 
 typedef struct {
   char    tableFname[TSDB_TABLE_FNAME_LEN];
@@ -356,6 +363,7 @@ typedef struct {
   int32_t pid;
   char    app[TSDB_APP_NAME_LEN];
   char    db[TSDB_DB_NAME_LEN];
+  int64_t startTime;
 } SConnectMsg;
 
 typedef struct SEpSet {
@@ -366,19 +374,17 @@ typedef struct SEpSet {
 } SEpSet;
 
 typedef struct {
-  int32_t acctId;
-  int32_t clusterId;
-  int32_t connId;
-  int8_t  superAuth;
-  int8_t  readAuth;
-  int8_t  writeAuth;
-  int8_t  reserved[5];
-  SEpSet  epSet;
+  int32_t  acctId;
+  uint32_t clusterId;
+  int32_t  connId;
+  int8_t   superUser;
+  int8_t   reserved[5];
+  SEpSet   epSet;
 } SConnectRsp;
 
 typedef struct {
   char    user[TSDB_USER_LEN];
-  char    pass[TSDB_KEY_LEN];
+  char    pass[TSDB_PASSWORD_LEN];
   int32_t maxUsers;
   int32_t maxDbs;
   int32_t maxTimeSeries;
@@ -393,7 +399,7 @@ typedef struct {
 
 typedef struct {
   char user[TSDB_USER_LEN];
-  char pass[TSDB_KEY_LEN];
+  char pass[TSDB_PASSWORD_LEN];
 } SCreateUserMsg, SAlterUserMsg;
 
 typedef struct {
@@ -668,7 +674,6 @@ typedef struct {
 
 typedef struct {
   int32_t statusInterval;
-  int32_t mnodeEqualVnodeNum;
   int64_t checkTime;                    // 1970-01-01 00:00:00.000
   char    timezone[TSDB_TIMEZONE_LEN];  // tsTimezone
   char    locale[TSDB_LOCALE_LEN];      // tsLocale
@@ -694,7 +699,7 @@ typedef struct SStatusMsg {
   int32_t     sver;
   int32_t     dnodeId;
   int32_t     clusterId;
-  uint32_t    rebootTime;  // time stamp for last reboot
+  int64_t     rebootTime;  // time stamp for last reboot
   int16_t     numOfCores;
   int16_t     numOfSupportMnodes;
   int16_t     numOfSupportVnodes;
@@ -770,7 +775,7 @@ typedef struct {
 
 typedef struct {
   char name[TSDB_TABLE_FNAME_LEN];
-} SStableInfoMsg;
+} SStbInfoMsg;
 
 typedef struct {
   SMsgHead  msgHead;
@@ -815,8 +820,8 @@ typedef struct {
 } SVgroupsMsg, SVgroupsInfo;
 
 typedef struct {
-  char       tableFname[TSDB_TABLE_FNAME_LEN];  // table id
-  char       stableFname[TSDB_TABLE_FNAME_LEN];
+  char       tbFname[TSDB_TABLE_FNAME_LEN];  // table id
+  char       stbFname[TSDB_TABLE_FNAME_LEN];
   int32_t    numOfTags;
   int32_t    numOfColumns;
   int8_t     precision;
@@ -880,7 +885,7 @@ typedef struct {
 
 typedef struct {
   int32_t dnodeId;
-  char    config[128];
+  char    config[TSDB_DNODE_CONFIG_LEN];
 } SCfgDnodeMsg;
 
 typedef struct {
@@ -973,8 +978,8 @@ typedef struct {
   char user[TSDB_USER_LEN];
   char spi;
   char encrypt;
-  char secret[TSDB_KEY_LEN];
-  char ckey[TSDB_KEY_LEN];
+  char secret[TSDB_PASSWORD_LEN];
+  char ckey[TSDB_PASSWORD_LEN];
 } SAuthMsg, SAuthRsp;
 
 typedef struct {

include/common/tdataformat.h

@@ -544,7 +544,7 @@ void   tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder);
 void   tdResetKVRowBuilder(SKVRowBuilder *pBuilder);
 SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder);
 
-static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, void *value) {
+static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, const void *value) {
   if (pBuilder->nCols >= pBuilder->tCols) {
     pBuilder->tCols *= 2;
     SColIdx* pColIdx = (SColIdx *)realloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols);

include/common/tep.h

+#ifndef TDENGINE_TEP_H
+#define TDENGINE_TEP_H
+
+#include "os.h"
+#include "taosmsg.h"
+
+typedef struct SCorEpSet {
+  int32_t version;
+  SEpSet  epSet;
+} SCorEpSet;
+
+int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port);
+bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2);
+
+void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet);
+
+#endif  // TDENGINE_TEP_H

include/common/tglobal.h

@@ -81,8 +81,6 @@ extern int64_t tsMaxRetentWindow;
 // db parameters in client
 extern int32_t tsCacheBlockSize;
 extern int32_t tsBlocksPerVnode;
-extern int32_t tsMinTablePerVnode;
-extern int32_t tsMaxTablePerVnode;
 extern int32_t tsTableIncStepPerVnode;
 extern int32_t tsMaxVgroupsPerDb;
 extern int16_t tsDaysPerFile;
@@ -108,22 +106,13 @@ extern int8_t  tsEnableBalance;
 extern int8_t  tsAlternativeRole;
 extern int32_t tsBalanceInterval;
 extern int32_t tsOfflineThreshold;
-extern int32_t tsMnodeEqualVnodeNum;
 extern int8_t  tsEnableFlowCtrl;
 extern int8_t  tsEnableSlaveQuery;
 extern int8_t  tsEnableAdjustMaster;
 
 // restful
-extern int8_t   tsEnableHttpModule;
 extern int32_t  tsRestRowLimit;
-extern uint16_t tsHttpPort;
-extern int32_t  tsHttpCacheSessions;
-extern int32_t  tsHttpSessionExpire;
-extern int32_t  tsHttpMaxThreads;
-extern int8_t   tsHttpEnableCompress;
-extern int8_t   tsHttpEnableRecordSql;
 extern int8_t   tsTelegrafUseFieldNum;
-extern int8_t   tsHttpDbNameMandatory;
 
 // mqtt
 extern int8_t tsEnableMqttModule;
@@ -145,7 +134,6 @@ extern int8_t tsEnableStream;
 
 // internal
 extern int8_t  tsPrintAuth;
-extern int8_t  tscEmbedded;
 extern char    tsVnodeDir[];
 extern char    tsMnodeDir[];
 extern int64_t tsTickPerDay[3];
@@ -194,7 +182,7 @@ extern SDiskCfg tsDiskCfg[];
 #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
 
 void    taosInitGlobalCfg();
-int32_t taosCheckGlobalCfg();
+int32_t taosCheckAndPrintCfg();
 int32_t taosCfgDynamicOptions(char *msg);
 bool    taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId);
 void    taosAddDataDir(int index, char *v1, int level, int primary);

include/common/ttime.h

@@ -44,10 +44,10 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision);
 int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision);
 int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision);
 
-int32_t parseAbsoluteDuration(char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision);
+int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision);
 int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision);
 
-int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth);
+int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth);
 void    deltaToUtcInitOnce();
 
 int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision);

include/common/tvariant.h

@@ -40,7 +40,7 @@ int32_t toInteger(const char* z, int32_t n, int32_t base, int64_t* value, bool*
 
 bool taosVariantIsValid(SVariant *pVar);
 
-void taosVariantCreate(SVariant *pVar, char* z, int32_t n, int32_t type);
+void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type);
 
 void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uint32_t type);
 

include/dnode/mgmt/dnode.h

@@ -33,7 +33,6 @@ typedef struct {
   int16_t  numOfSupportQnodes;
   int8_t   enableTelem;
   int32_t  statusInterval;
-  int32_t  mnodeEqualVnodeNum;
   float    numOfThreadsPerCore;
   float    ratioOfQueryCores;
   int32_t  maxShellConns;

include/dnode/mnode/mnode.h

@@ -47,7 +47,6 @@ typedef struct SMnodeCfg {
   int32_t sver;
   int8_t  enableTelem;
   int32_t statusInterval;
-  int32_t mnodeEqualVnodeNum;
   int32_t shellActivityTimer;
   char   *timezone;
   char   *locale;

include/dnode/mnode/sdb/sdb.h

@@ -158,8 +158,8 @@ typedef enum {
   SDB_USER = 5,
   SDB_AUTH = 6,
   SDB_ACCT = 7,
-  SDB_VGROUP = 9,
-  SDB_STABLE = 9,
+  SDB_VGROUP = 8,
+  SDB_STB = 9,
   SDB_DB = 10,
   SDB_FUNC = 11,
   SDB_MAX = 12

include/dnode/vnode/tq/tq.h

@@ -256,7 +256,7 @@ typedef struct STQ {
   // the collection of group handle
   // the handle of kvstore
   char*  path;
-  STqCfg*    tqConfig;
+  STqCfg*      tqConfig;
   TqLogReader* tqLogReader; 
   TqMemRef     tqMemRef;
   TqMetaStore* tqMeta;

include/dnode/vnode/vnode.h

@@ -36,6 +36,8 @@ typedef struct SVnodeCfg {
   struct {
     /** write buffer size */
     uint64_t wsize;
+    uint64_t ssize;
+    uint64_t lsize;
     /** use heap allocator or arena allocator */
     bool isHeapAllocator;
   };

include/libs/catalog/catalog.h

@@ -44,12 +44,12 @@ typedef struct SCatalogReq {
   bool    qNodeRequired;  // valid qnode
 } SCatalogReq;
 
-typedef struct SCatalogRsp {
+typedef struct SMetaData {
   SArray    *pTableMeta;  // STableMeta array
   SArray    *pVgroupInfo; // SVgroupInfo list
   SArray    *pUdfList;    // udf info list
   SEpSet    *pEpSet;      // qnode epset list
-} SCatalogRsp;
+} SMetaData;
 
 typedef struct STableComInfo {
   uint8_t numOfTags;      // the number of tags in schema
@@ -135,7 +135,7 @@ int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSe
  * @param pMetaData
  * @return
  */
-int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SCatalogRsp* pRsp);
+int32_t catalogGetAllMeta(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp);
 
 
 int32_t catalogGetQnodeList(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, SEpSet* pQnodeEpSet);

include/libs/parser/parser.h

@@ -131,6 +131,18 @@ struct SInsertStmtInfo;
  */
 bool qIsInsertSql(const char* pStr, size_t length);
 
+typedef struct SParseContext {
+  const char* pSql;    // sql string
+  size_t sqlLen;       // length of the sql string
+  int64_t id;          // operator id, generated by uuid generator
+  const char* pDbname;
+  const SEpSet* pEpSet; 
+  int8_t schemaAttached; // denote if submit block is built with table schema or not
+
+  char* pMsg;           // extended error message if exists to help avoid the problem in sql statement.
+  int32_t msgLen;      // max length of the msg
+} SParseContext;
+
 /**
  * Parse the sql statement and then return the SQueryStmtInfo as the result of bounded AST.
  * @param pSql     sql string
@@ -141,16 +153,35 @@ bool qIsInsertSql(const char* pStr, size_t length);
  */
 int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** pQueryInfo, int64_t id, char* msg, int32_t msgLen);
 
+typedef enum {
+  PAYLOAD_TYPE_KV = 0,
+  PAYLOAD_TYPE_RAW = 1,
+} EPayloadType;
+
+typedef struct SVgDataBlocks {
+  int64_t     vgId;         // virtual group id
+  int32_t     numOfTables;  // number of tables in current submit block
+  uint32_t    size;
+  char       *pData;        // SMsgDesc + SSubmitMsg + SSubmitBlk + ...
+} SVgDataBlocks;
+
+typedef struct SInsertStmtInfo {
+  SArray*     pDataBlocks;         // data block for each vgroup, SArray<SVgDataBlocks*>.
+  int8_t      schemaAttache;        // denote if submit block is built with table schema or not
+  uint8_t     payloadType;         // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
+  uint32_t    insertType;          // insert data from [file|sql statement| bound statement]
+  const char* sql;                 // current sql statement position
+} SInsertStmtInfo;
+
 /**
  * Parse the insert sql statement.
  * @param pStr            sql string
  * @param length          length of the sql string
- * @param pInsertParam    data in binary format to submit to vnode directly.
  * @param id              operator id, generated by uuid generator.
  * @param msg             extended error message if exists to help avoid the problem in sql statement.
- * @return
+ * @return                data in binary format to submit to vnode directly.
  */
-int32_t qParseInsertSql(const char* pStr, size_t length, struct SInsertStmtInfo** pInsertInfo, int64_t id, char* msg, int32_t msgLen);
+ int32_t qParseInsertSql(SParseContext* pContext, struct SInsertStmtInfo** pInfo);
 
 /**
  * Convert a normal sql statement to only query tags information to enable that the subscribe client can be aware quickly of the true vgroup ids that

include/libs/transport/trpc.h

@@ -29,11 +29,6 @@ extern "C" {
 
 extern int tsRpcHeadSize;
 
-typedef struct SRpcCorEpSet {
-  int32_t version; 
-  SEpSet epSet; 
-} SRpcCorEpSet;
-
 typedef struct SRpcConnInfo {
   uint32_t  clientIp;
   uint16_t  clientPort;

include/libs/wal/wal.h

@@ -32,12 +32,49 @@ extern int32_t wDebugFlag;
 #define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }}
 #define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }}
 
+#define WAL_PREFIX       "wal"
+#define WAL_PREFIX_LEN   3
+#define WAL_NOSUFFIX_LEN 20
+#define WAL_SUFFIX_AT    (WAL_NOSUFFIX_LEN+1)
+#define WAL_LOG_SUFFIX   "log"
+#define WAL_INDEX_SUFFIX "idx"
+#define WAL_REFRESH_MS   1000
+#define WAL_MAX_SIZE     (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + 16)
+#define WAL_PATH_LEN     (TSDB_FILENAME_LEN + 12)
+#define WAL_FILE_LEN     (WAL_PATH_LEN + 32)
+
+#define WAL_IDX_ENTRY_SIZE    (sizeof(int64_t)*2)
+#define WAL_CUR_POS_WRITABLE  1
+#define WAL_CUR_FILE_WRITABLE 2
+#define WAL_CUR_FAILED        4
+
+#pragma pack(push,1)
 typedef enum {
   TAOS_WAL_NOLOG = 0,
   TAOS_WAL_WRITE = 1,
   TAOS_WAL_FSYNC = 2
 } EWalType;
 
+typedef struct SWalReadHead {
+  int8_t   sver;
+  uint8_t  msgType;
+  int8_t   reserved[2];
+  int32_t  len;
+  //int64_t  ingestTs;  //not implemented
+  int64_t  version;
+  char     body[];
+} SWalReadHead;
+
+typedef struct {
+  int32_t  vgId;
+  int32_t  fsyncPeriod;      // millisecond
+  int32_t  retentionPeriod;  // secs
+  int32_t  rollPeriod;       // secs
+  int64_t  retentionSize;
+  int64_t  segSize;
+  EWalType level;         // wal level
+} SWalCfg;
+
 typedef struct {
   //union {
     //uint32_t info;
@@ -47,87 +84,53 @@ typedef struct {
       //uint32_t reserved : 24;
     //};
   //};
-  int8_t   sver;
-  uint8_t  msgType;
-  int8_t   reserved[2];
-  int32_t  len;
-  int64_t  version;
-  uint32_t signature;
   uint32_t cksumHead;
   uint32_t cksumBody;
-  char     cont[];
+  SWalReadHead head;
 } SWalHead;
 
-typedef struct {
-  int32_t  vgId;
-  int32_t  fsyncPeriod;  // millisecond
-  int32_t  rollPeriod;
-  int64_t  segSize;
-  EWalType walLevel;     // wal level
-} SWalCfg;
-
-#define WAL_PREFIX       "wal"
-#define WAL_PREFIX_LEN   3
-#define WAL_NOSUFFIX_LEN 20
-#define WAL_SUFFIX_AT    (WAL_NOSUFFIX_LEN+1)
-#define WAL_LOG_SUFFIX   "log"
-#define WAL_INDEX_SUFFIX "idx"
-#define WAL_REFRESH_MS   1000
-#define WAL_MAX_SIZE     (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + 16)
-#define WAL_SIGNATURE    ((uint32_t)(0xFAFBFDFEUL))
-#define WAL_PATH_LEN     (TSDB_FILENAME_LEN + 12)
-#define WAL_FILE_LEN     (WAL_PATH_LEN + 32)
-//#define WAL_FILE_NUM     1 // 3
-#define WAL_FILESET_MAX  128
-
-#define WAL_IDX_ENTRY_SIZE     (sizeof(int64_t)*2)
-#define WAL_CUR_POS_WRITABLE  1
-#define WAL_CUR_FILE_WRITABLE 2
-#define WAL_CUR_FAILED        4
+typedef struct SWalVer {
+  int64_t firstVer;
+  int64_t verInSnapshotting;
+  int64_t snapshotVer;
+  int64_t commitVer;
+  int64_t lastVer;
+} SWalVer;
 
 typedef struct SWal {
   // cfg
-  int32_t  vgId;
-  int32_t  fsyncPeriod;  // millisecond
-  int32_t  rollPeriod;  // second
-  int64_t  segSize;
-  int64_t  retentionSize;
-  int32_t  retentionPeriod;
-  EWalType level;
-  //total size
-  int64_t  totSize;
-  //fsync seq
-  int32_t  fsyncSeq;
-  //reference
-  int64_t refId;
-  //write tfd
-  int64_t writeLogTfd;
-  int64_t writeIdxTfd;
-  //read tfd
-  int64_t readLogTfd;
-  int64_t readIdxTfd;
-  //current version
-  int64_t curVersion;
-  //wal lifecycle
-  int64_t firstVersion;
-  int64_t snapshotVersion;
-  int64_t commitVersion;
-  int64_t lastVersion;
-  //roll status
-  int64_t lastRollSeq;
+  SWalCfg cfg;
+  SWalVer vers;
   //file set
   int32_t writeCur;
-  int32_t readCur;
+  int64_t writeLogTfd;
+  int64_t writeIdxTfd;
   SArray* fileInfoSet;
   //ctl
   int32_t curStatus;
+  int32_t fsyncSeq;
+  int64_t totSize;
+  int64_t refId;
+  int64_t lastRollSeq;
   pthread_mutex_t mutex;
   //path
   char path[WAL_PATH_LEN];
   //reusable write head
-  SWalHead head;
+  SWalHead writeHead;
 } SWal;  // WAL HANDLE
 
+typedef struct SWalReadHandle {
+  SWal*        pWal;
+  int64_t      readLogTfd;
+  int64_t      readIdxTfd;
+  int64_t      curFileFirstVer;
+  int64_t      curVersion;
+  int64_t      capacity;
+  int64_t      status;  //if cursor valid
+  SWalHead*    pHead;
+} SWalReadHandle;
+#pragma pack(pop)
+
 typedef int32_t (*FWalWrite)(void *ahandle, void *pHead);
 
 // module initialization
@@ -148,10 +151,15 @@ int32_t walCommit(SWal *, int64_t ver);
 // truncate after
 int32_t walRollback(SWal *, int64_t ver);
 // notify that previous logs can be pruned safely
-int32_t walTakeSnapshot(SWal *, int64_t ver);
+int32_t walBeginTakeSnapshot(SWal *, int64_t ver);
+int32_t walEndTakeSnapshot(SWal *);
 //int32_t  walDataCorrupted(SWal*);
 
 // read
+SWalReadHandle* walOpenReadHandle(SWal *);
+void    walCloseReadHandle(SWalReadHandle *);
+int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver);
+
 int32_t walRead(SWal *, SWalHead **, int64_t ver);
 int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum);
 

include/os/osSemaphore.h

@@ -57,7 +57,7 @@ int64_t taosGetPthreadId(pthread_t thread);
 void    taosResetPthread(pthread_t* thread);
 bool    taosComparePthread(pthread_t first, pthread_t second);
 int32_t taosGetPId();
-int32_t taosGetCurrentAPPName(char* name, int32_t* len);
+int32_t taosGetAppName(char* name, int32_t* len);
 
 #ifdef __cplusplus
 }

include/os/osString.h

@@ -38,11 +38,11 @@ extern "C" {
     (dst)[(size)-1] = 0;            \
   } while (0)
 
-int64_t taosStr2int64(char *str);
+int64_t taosStr2int64(const char *str);
 
 // USE_LIBICONV
 int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs);
-bool    taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len);
+bool    taosMbsToUcs4(const char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len);
 int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize);
 bool    taosValidateEncodec(const char *encodec);
 char *  taosCharsetReplace(char *charsetstr);

include/util/taoserror.h

@@ -153,22 +153,23 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_SDB_INVALID_DATA_LEN          TAOS_DEF_ERROR_CODE(0, 0x032A)
 #define TSDB_CODE_SDB_INVALID_DATA_CONTENT      TAOS_DEF_ERROR_CODE(0, 0x032B)
 
-#define TSDB_CODE_MND_DNODE_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x0330)  //"DNode already exists")
-#define TSDB_CODE_MND_DNODE_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x0331)  //"DNode does not exist")
+// mnode-dnode
+#define TSDB_CODE_MND_DNODE_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x0330)
+#define TSDB_CODE_MND_DNODE_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x0331)
+#define TSDB_CODE_MND_NO_ENOUGH_DNODES          TAOS_DEF_ERROR_CODE(0, 0x0332)
+#define TSDB_CODE_MND_INVALID_CLUSTER_CFG       TAOS_DEF_ERROR_CODE(0, 0x0333)
+#define TSDB_CODE_MND_INVALID_CLUSTER_ID        TAOS_DEF_ERROR_CODE(0, 0x0334)
+#define TSDB_CODE_MND_INVALID_DNODE_CFG         TAOS_DEF_ERROR_CODE(0, 0x0335)
+#define TSDB_CODE_MND_INVALID_DNODE_EP          TAOS_DEF_ERROR_CODE(0, 0x0336)
+#define TSDB_CODE_MND_INVALID_DNODE_ID          TAOS_DEF_ERROR_CODE(0, 0x0337)
+
+// mnode-vgroup
 #define TSDB_CODE_MND_VGROUP_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x0332)  //"VGroup does not exist")
-#define TSDB_CODE_MND_NO_REMOVE_MASTER          TAOS_DEF_ERROR_CODE(0, 0x0333)  //"Master DNode cannot be removed")
-#define TSDB_CODE_MND_NO_ENOUGH_DNODES          TAOS_DEF_ERROR_CODE(0, 0x0334)  //"Out of DNodes")
-#define TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT  TAOS_DEF_ERROR_CODE(0, 0x0335)  //"Cluster cfg inconsistent")
-#define TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION  TAOS_DEF_ERROR_CODE(0, 0x0336)  //"Invalid dnode cfg option")
-#define TSDB_CODE_MND_BALANCE_ENABLED           TAOS_DEF_ERROR_CODE(0, 0x0337)  //"Balance already enabled")
 #define TSDB_CODE_MND_VGROUP_NOT_IN_DNODE       TAOS_DEF_ERROR_CODE(0, 0x0338)  //"Vgroup not in dnode")
 #define TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE   TAOS_DEF_ERROR_CODE(0, 0x0339)  //"Vgroup already in dnode")
-#define TSDB_CODE_MND_DNODE_NOT_FREE            TAOS_DEF_ERROR_CODE(0, 0x033A)  //"Dnode not avaliable")
-#define TSDB_CODE_MND_INVALID_CLUSTER_ID        TAOS_DEF_ERROR_CODE(0, 0x033B)  //"Cluster id not match")
 #define TSDB_CODE_MND_NOT_READY                 TAOS_DEF_ERROR_CODE(0, 0x033C)  //"Cluster not ready")
-#define TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED   TAOS_DEF_ERROR_CODE(0, 0x033D)  //"Dnode Id not configured")
-#define TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED   TAOS_DEF_ERROR_CODE(0, 0x033E)  //"Dnode Ep not configured")
 
+// mnode-acct
 #define TSDB_CODE_MND_ACCT_ALREADY_EXIST        TAOS_DEF_ERROR_CODE(0, 0x0340)  //"Account already exists")
 #define TSDB_CODE_MND_ACCT_NOT_EXIST            TAOS_DEF_ERROR_CODE(0, 0x0341)  //"Invalid account")
 #define TSDB_CODE_MND_INVALID_ACCT_OPTION       TAOS_DEF_ERROR_CODE(0, 0x0342)  //"Invalid account options")
@@ -184,8 +185,18 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_MNODE_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x0348)  //"Mnode already exists")
 #define TSDB_CODE_MND_MNODE_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x0349)  //"Mnode not there")
 
-// mnode-table
-#define TSDB_CODE_MND_TABLE_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x0360)  //"Table already exists")
+// mnode-stable
+#define TSDB_CODE_MND_STB_NOT_EXIST             TAOS_DEF_ERROR_CODE(0, 0x0360)
+#define TSDB_CODE_MND_STB_ALREADY_EXIST         TAOS_DEF_ERROR_CODE(0, 0x0360)
+#define TSDB_CODE_MND_STB_INVALID_IGEXIST       TAOS_DEF_ERROR_CODE(0, 0x0360)
+#define TSDB_CODE_MND_STB_INVALID_COLS_NUM      TAOS_DEF_ERROR_CODE(0, 0x0360)
+#define TSDB_CODE_MND_STB_INVALID_TAGS_NUM      TAOS_DEF_ERROR_CODE(0, 0x0360)
+#define TSDB_CODE_MND_STB_INVALID_COL_TYPE      TAOS_DEF_ERROR_CODE(0, 0x0360)
+#define TSDB_CODE_MND_STB_INVALID_COL_ID        TAOS_DEF_ERROR_CODE(0, 0x0360)
+#define TSDB_CODE_MND_STB_INVALID_COL_BYTES     TAOS_DEF_ERROR_CODE(0, 0x0360)
+#define TSDB_CODE_MND_STB_INVALID_COL_NAME      TAOS_DEF_ERROR_CODE(0, 0x0360)
+
+
 #define TSDB_CODE_MND_INVALID_TABLE_ID          TAOS_DEF_ERROR_CODE(0, 0x0361)  //"Table name too long")
 #define TSDB_CODE_MND_INVALID_TABLE_NAME        TAOS_DEF_ERROR_CODE(0, 0x0362)  //"Table does not exist")
 #define TSDB_CODE_MND_INVALID_TABLE_TYPE        TAOS_DEF_ERROR_CODE(0, 0x0363)  //"Invalid table type in tsdb")

include/util/tarray.h

@@ -43,6 +43,13 @@ typedef struct SArray {
  */
 void* taosArrayInit(size_t size, size_t elemSize);
 
+/**
+ *
+ * @param tsize
+ * @return
+ */
+int32_t taosArrayEnsureCap(SArray* pArray, size_t tsize);
+
 /**
  *
  * @param pArray
@@ -153,6 +160,13 @@ void taosArraySet(SArray* pArray, size_t index, void* pData);
  */
 void taosArrayPopFrontBatch(SArray* pArray, size_t cnt);
 
+/**
+ * remove some data entry from front
+ * @param pArray
+ * @param cnt
+ */
+void taosArrayPopTailBatch(SArray* pArray, size_t cnt);
+
 /**
  * remove data entry of the given index
  * @param pArray
@@ -213,6 +227,14 @@ void taosArraySortString(SArray* pArray, __compar_fn_t comparFn);
  */
 void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags);
 
+/**
+ * search the array, return index of the element
+ * @param pArray
+ * @param compar
+ * @param key
+ */
+int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags);
+
 /**
  * search the array
  * @param pArray

include/util/tchecksum.h

@@ -39,7 +39,7 @@ static FORCE_INLINE int taosCalcChecksumAppend(TSCKSUM csi, uint8_t *stream, uin
 }
 
 static FORCE_INLINE int taosCheckChecksum(const uint8_t *stream, uint32_t ssize, TSCKSUM checksum) {
-  return (checksum == (*crc32c)(0, stream, (size_t)ssize));
+  return (checksum != (*crc32c)(0, stream, (size_t)ssize));
 }
 
 static FORCE_INLINE int taosCheckChecksumWhole(const uint8_t *stream, uint32_t ssize) {

include/util/tconfig.h

@@ -20,7 +20,7 @@
 extern "C" {
 #endif
 
-#define TSDB_CFG_MAX_NUM    123
+#define TSDB_CFG_MAX_NUM    115
 #define TSDB_CFG_PRINT_LEN  23
 #define TSDB_CFG_OPTION_LEN 24
 #define TSDB_CFG_VALUE_LEN  41
@@ -83,11 +83,11 @@ extern int32_t    tsGlobalConfigNum;
 extern char *     tsCfgStatusStr[];
 
 void    taosReadGlobalLogCfg();
-int32_t taosReadGlobalCfg();
-void    taosPrintGlobalCfg();
+int32_t taosReadCfgFromFile();
+void    taosPrintCfg();
 void    taosDumpGlobalCfg();
 
-void        taosInitConfigOption(SGlobalCfg cfg);
+void        taosAddConfigOption(SGlobalCfg cfg);
 SGlobalCfg *taosGetConfigOption(const char *option);
 
 #ifdef __cplusplus

include/util/tdef.h

@@ -161,7 +161,7 @@ do { \
 
 #define TSDB_NODE_NAME_LEN        64
 #define TSDB_TABLE_NAME_LEN       193     // it is a null-terminated string
-#define TSDB_DB_NAME_LEN          33
+#define TSDB_DB_NAME_LEN          65
 #define TSDB_FULL_DB_NAME_LEN     (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN)
 
 #define TSDB_FUNC_NAME_LEN         65
@@ -193,7 +193,7 @@ do { \
 #define TSDB_MAX_TAG_CONDITIONS   1024
 
 #define TSDB_AUTH_LEN             16
-#define TSDB_KEY_LEN              16
+#define TSDB_PASSWORD_LEN         32
 #define TSDB_VERSION_LEN          12
 #define TSDB_LABEL_LEN            8 
 
@@ -209,6 +209,8 @@ do { \
 #define TSDB_STEP_NAME_LEN        32
 #define TSDB_STEP_DESC_LEN        128
 
+#define TSDB_DNODE_CONFIG_LEN     128
+
 #define TSDB_MQTT_HOSTNAME_LEN    64
 #define TSDB_MQTT_PORT_LEN        8
 #define TSDB_MQTT_USER_LEN        24

include/util/tdlist.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_UTIL_TDLIST_H_
+#define _TD_UTIL_TDLIST_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Single linked list
+#define TD_SLIST_NODE(TYPE) \
+  struct {                  \
+    struct TYPE *sl_next_;  \
+  }
+
+#define TD_SLIST(TYPE)      \
+  struct {                  \
+    struct TYPE *sl_head_;  \
+    int          sl_neles_; \
+  }
+
+#define TD_SLIST_HEAD(sl) ((sl)->sl_head_)
+#define TD_SLIST_NELES(sl) ((sl)->sl_neles_)
+#define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_)
+
+#define tSListInit(sl)     \
+  do {                     \
+    (sl)->sl_head_ = NULL; \
+    (sl)->sl_neles_ = 0;   \
+  } while (0)
+
+#define tSListPush(sl, sln)                      \
+  do {                                           \
+    TD_SLIST_NODE_NEXT(sln) = TD_SLIST_HEAD(sl); \
+    TD_SLIST_HEAD(sl) = (sln);                   \
+    TD_SLIST_NELES(sl) += 1;                     \
+  } while (0)
+
+#define tSListPop(sl)                                          \
+  do {                                                         \
+    TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT(TD_SLIST_HEAD(sl)); \
+    TD_SLIST_NELES(sl) -= 1;                                   \
+  } while (0)
+
+// Double linked list
+#define TD_DLIST_NODE(TYPE) \
+  struct {                  \
+    TYPE *dl_prev_;         \
+    TYPE *dl_next_;         \
+  }
+
+#define TD_DLIST(TYPE)      \
+  struct {                  \
+    struct TYPE *dl_head_;  \
+    struct TYPE *dl_tail_;  \
+    int          dl_neles_; \
+  }
+
+#define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_)
+#define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_)
+#define TD_DLIST_HEAD(dl) ((dl)->dl_head_)
+#define TD_DLIST_TAIL(dl) ((dl)->dl_tail_)
+#define TD_DLIST_NELES(dl) ((dl)->dl_neles_)
+
+#define tDListInit(dl)                            \
+  do {                                            \
+    TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = NULL; \
+    TD_DLIST_NELES(dl) = 0;                       \
+  } while (0)
+
+#define tDListAppend(dl, dln)                                   \
+  do {                                                          \
+    if (TD_DLIST_HEAD(dl) == NULL) {                            \
+      TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \
+      TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln);            \
+    } else {                                                    \
+      TD_DLIST_NODE_PREV(dln) = TD_DLIST_TAIL(dl);              \
+      TD_DLIST_NODE_NEXT(dln) = NULL;                           \
+      TD_DLIST_NODE_NEXT(TD_DLIST_TAIL(dl)) = (dln);            \
+      TD_DLIST_TAIL(dl) = (dln);                                \
+    }                                                           \
+    TD_DLIST_NELES(dl) += 1;                                    \
+  } while (0)
+
+#define tDListPrepend(dl, dln)                                  \
+  do {                                                          \
+    if (TD_DLIST_HEAD(dl) == NULL) {                            \
+      TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \
+      TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln);            \
+    } else {                                                    \
+      TD_DLIST_NODE_PREV(dln) = NULL;                           \
+      TD_DLIST_NODE_NEXT(dln) = TD_DLIST_HEAD(dl);              \
+      TD_DLIST_NODE_PREV(TD_DLIST_HEAD(dl)) = (dln);            \
+      TD_DLIST_HEAD(dl) = (dln);                                \
+    }                                                           \
+    TD_DLIST_NELES(dl) += 1;                                    \
+  } while (0)
+
+#define tDListPop(dl, dln)                                                   \
+  do {                                                                       \
+    if (TD_DLIST_HEAD(dl) == (dln)) {                                        \
+      TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT(dln);                           \
+    }                                                                        \
+    if (TD_DLIST_TAIL(dl) == (dln)) {                                        \
+      TD_DLIST_TAIL(dl) = TD_DLIST_NODE_PREV(dln);                           \
+    }                                                                        \
+    if (TD_DLIST_NODE_PREV(dln) != NULL) {                                   \
+      TD_DLIST_NODE_NEXT(TD_DLIST_NODE_PREV(dln)) = TD_DLIST_NODE_NEXT(dln); \
+    }                                                                        \
+    if (TD_DLIST_NODE_NEXT(dln) != NULL) {                                   \
+      TD_DLIST_NODE_PREV(TD_DLIST_NODE_NEXT(dln)) = TD_DLIST_NODE_PREV(dln); \
+    }                                                                        \
+    TD_DLIST_NELES(dl) -= 1;                                                 \
+    TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL;                \
+  } while (0)
+
+#if 0
+// List iterator
+#define TD_LIST_FITER 0
+#define TD_LIST_BITER 1
+#define TD_LIST_ITER(S)     \
+  struct {                  \
+    int it_dir_;            \
+    S * it_next_;           \
+    S * it_ptr_;            \
+    TD_DLIST(S) * it_list_; \
+  }
+
+#define tlistIterInit(it, l, dir)   \
+  (it)->it_dir_ = (dir);            \
+  (it)->it_list_ = l;               \
+  if ((dir) == TD_LIST_FITER) {     \
+    (it)->it_next_ = (l)->dl_head_; \
+  } else {                          \
+    (it)->it_next_ = (l)->dl_tail_; \
+  }
+
+#define tlistIterNext(it)                       \
+  ({                                            \
+    (it)->it_ptr_ = (it)->it_next_;             \
+    if ((it)->it_next_ != NULL) {               \
+      if ((it)->it_dir_ == TD_LIST_FITER) {     \
+        (it)->it_next_ = (it)->it_next_->next_; \
+      } else {                                  \
+        (it)->it_next_ = (it)->it_next_->prev_; \
+      }                                         \
+    }                                           \
+    (it)->it_ptr_;                              \
+  })
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_TDLIST_H_*/
\ No newline at end of file

include/util/tmacro.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_UTIL_MACRO_H_
+#define _TD_UTIL_MACRO_H_
+
+#include "os.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Module init/clear MACRO definitions
+#define TD_MOD_UNINITIALIZED 0
+#define TD_MOD_INITIALIZED 1
+
+#define TD_MOD_UNCLEARD 0
+#define TD_MOD_CLEARD 1
+
+#define TD_DEF_MOD_INIT_FLAG(MOD) static int8_t MOD##InitFlag = TD_MOD_UNINITIALIZED
+#define TD_DEF_MOD_CLEAR_FLAG(MOD) static int8_t MOD##ClearFlag = TD_MOD_UNCLEARD
+
+#define TD_CHECK_AND_SET_MODE_INIT(MOD) \
+  atomic_val_compare_exchange_8(&(MOD##InitFlag), TD_MOD_UNINITIALIZED, TD_MOD_INITIALIZED)
+
+#define TD_CHECK_AND_SET_MOD_CLEAR(MOD) atomic_val_compare_exchange_8(&(MOD##ClearFlag), TD_MOD_UNCLEARD, TD_MOD_CLEARD)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_MACRO_H_*/
\ No newline at end of file

include/util/tutil.h

@@ -29,7 +29,7 @@ int32_t strdequote(char *src);
 int32_t strndequote(char *dst, const char *z, int32_t len);
 int32_t strRmquote(char *z, int32_t len);
 size_t  strtrim(char *src);
-char   *strnchr(char *haystack, char needle, int32_t len, bool skipquote);
+char   *strnchr(const char *haystack, char needle, int32_t len, bool skipquote);
 char  **strsplit(char *src, const char *delim, int32_t *num);
 char   *strtolower(char *dst, const char *src);
 char   *strntolower(char *dst, const char *src, int32_t n);
@@ -45,14 +45,25 @@ char    *taosIpStr(uint32_t ipInt);
 uint32_t ip2uint(const char *const ip_addr);
 void     taosIp2String(uint32_t ip, char *str);
 void     taosIpPort2String(uint32_t ip, uint16_t port, char *str);
-int32_t  taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port);
 
 static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) {
   T_MD5_CTX context;
   tMD5Init(&context);
   tMD5Update(&context, inBuf, (unsigned int)inLen);
   tMD5Final(&context);
-  memcpy(target, context.digest, TSDB_KEY_LEN);
+  memcpy(target, context.digest, tListLen(context.digest));
+}
+
+static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *target) {
+  T_MD5_CTX context;
+  tMD5Init(&context);
+  tMD5Update(&context, inBuf, (unsigned int)len);
+  tMD5Final(&context);
+
+  sprintf(target, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], context.digest[1], context.digest[2],
+          context.digest[3], context.digest[4], context.digest[5], context.digest[6], context.digest[7],
+          context.digest[8], context.digest[9], context.digest[10], context.digest[11], context.digest[12],
+          context.digest[13], context.digest[14], context.digest[15]);
 }
 
 #ifdef __cplusplus

source/client/CMakeLists.txt

@@ -2,10 +2,14 @@ aux_source_directory(src CLIENT_SRC)
 add_library(taos ${CLIENT_SRC})
 target_include_directories(
     taos
-    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+        PUBLIC "${CMAKE_SOURCE_DIR}/include/client"
+        PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 target_link_libraries(
     taos
     PRIVATE common
     INTERFACE api
+    PRIVATE os util common transport parser
 )
+
+ADD_SUBDIRECTORY(test)
\ No newline at end of file

source/client/inc/clientInt.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_CLIENTINT_H
+#define TDENGINE_CLIENTINT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "taos.h"
+#include "taosmsg.h"
+#include "thash.h"
+#include "tlist.h"
+#include "trpc.h"
+#include "tdef.h"
+#include "tmsgtype.h"
+#include "tep.h"
+
+typedef struct SQueryExecMetric {
+  int64_t      start;    // start timestamp
+  int64_t      parsed;   // start to parse
+  int64_t      send;     // start to send to server
+  int64_t      rsp;      // receive response from server
+} SQueryExecMetric;
+
+typedef struct SInstanceActivity {
+  uint64_t     numOfInsertsReq;
+  uint64_t     numOfInsertRows;
+  uint64_t     insertElapsedTime;
+  uint64_t     insertBytes;         // submit to tsdb since launched.
+
+  uint64_t     fetchBytes;
+  uint64_t     queryElapsedTime;
+  uint64_t     numOfSlowQueries;
+  uint64_t     totalRequests;
+  uint64_t     currentRequests;      // the number of SRequestObj
+} SInstanceActivity;
+
+typedef struct SHeartBeatInfo {
+  void  *pTimer;   // timer, used to send request msg to mnode
+} SHeartBeatInfo;
+
+typedef struct SAppInstInfo {
+  int64_t           numOfConns;
+  SCorEpSet         mgmtEp;
+  SInstanceActivity summary;
+  SList            *pConnList;  // STscObj linked list
+  uint32_t          clusterId;
+  void             *pTransporter;
+} SAppInstInfo;
+
+typedef struct SAppInfo {
+  int64_t        startTime;
+  char           appName[TSDB_APP_NAME_LEN];
+  char          *ep;
+  int32_t        pid;
+  int32_t        numOfThreads;
+  SHeartBeatInfo hb;
+  SHashObj      *pInstMap;
+} SAppInfo;
+
+typedef struct STscObj {
+  char               user[TSDB_USER_LEN];
+  char               pass[TSDB_PASSWORD_LEN];
+  char               acctId[TSDB_ACCT_ID_LEN];
+  char               db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN];
+  uint32_t           connId;
+  uint64_t           id;       // ref ID returned by taosAddRef
+//  struct SSqlObj    *sqlList;
+  void              *pTransporter;
+  pthread_mutex_t    mutex;     // used to protect the operation on db
+  int32_t            numOfReqs; // number of sqlObj from this tscObj
+  SAppInstInfo      *pAppInfo;
+} STscObj;
+
+typedef struct SReqBody {
+  tsem_t    rspSem;        // not used now
+  void*     fp;
+  void*     param;
+} SRequestBody;
+
+typedef struct SRequestObj {
+  uint64_t         requestId;
+  int32_t          type;   // request type
+  STscObj         *pTscObj;
+  SQueryExecMetric metric;
+  char            *sqlstr;  // sql string
+  SRequestBody     body;
+  int64_t          self;
+  char            *msgBuf;
+  int32_t          code;
+  void            *pInfo;   // sql parse info, generated by parser module
+} SRequestObj;
+
+typedef struct SRequestMsgBody {
+  int32_t  msgType;
+  void    *pData;
+  int32_t  msgLen;
+  uint64_t requestId;
+  uint64_t requestObjRefId;
+} SRequestMsgBody;
+
+extern SAppInfo   appInfo;
+extern int32_t    tscReqRef;
+extern void      *tscQhandle;
+extern int32_t    tscConnRef;
+
+extern int (*tscBuildMsg[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsg);
+extern int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen);
+
+int   taos_init();
+
+void* createTscObj(const char* user, const char* auth, const char *ip, uint32_t port, SAppInstInfo* pAppInfo);
+void  destroyTscObj(void*pObj);
+
+void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type);
+void destroyRequest(SRequestObj* pRequest);
+
+TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port);
+
+void taos_init_imp(void);
+int taos_options_imp(TSDB_OPTION option, const char *str);
+
+void* openTransporter(const char *user, const char *auth);
+
+void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet);
+void initMsgHandleFp();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_CLIENTINT_H

source/client/inc/tscLog.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TSCLOG_H
+#define TDENGINE_TSCLOG_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "tlog.h"
+
+#define tscFatal(...)  do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", cDebugFlag, __VA_ARGS__); }} while(0)
+#define tscError(...)  do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", cDebugFlag, __VA_ARGS__); }} while(0)
+#define tscWarn(...)   do { if (cDebugFlag & DEBUG_WARN)  { taosPrintLog("TSC WARN ", cDebugFlag, __VA_ARGS__); }}  while(0)
+#define tscInfo(...)   do { if (cDebugFlag & DEBUG_INFO)  { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0)
+#define tscDebug(...)  do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0)
+#define tscTrace(...)  do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0)
+#define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} while(0)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

source/client/src/client.c

@@ -13,14 +13,51 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-//#include "taos.h"
+#include "os.h"
+#include "tdef.h"
+#include "tglobal.h"
+#include "clientInt.h"
+#include "tscLog.h"
 
-//TAOS_RES *taos_query(TAOS *taos, const char *sql) {
-//
-//}
-#include "taosmsg.h"
+TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) {
+  int32_t p = (port != 0)? port:tsServerPort;
 
-int  taos_init() { return 0; }
-void taos_cleanup(void) {}
+  tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db);
+  if (user == NULL) {
+    user = TSDB_DEFAULT_USER;
+  }
 
+  if (pass == NULL) {
+    pass = TSDB_DEFAULT_PASS;
+  }
+
+  return taos_connect_internal(ip, user, pass, NULL, db, p);
+}
+
+TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port) {
+  tscDebug("try to connect to %s:%u by auth, user:%s db:%s", ip, port, user, db);
+  if (user == NULL) {
+    user = TSDB_DEFAULT_USER;
+  }
+
+  if (auth == NULL) {
+    tscError("No auth info is given, failed to connect to server");
+    return NULL;
+  }
+
+  return taos_connect_internal(ip, user, NULL, auth, db, port);
+}
+
+TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port) {
+  char ipStr[TSDB_EP_LEN]      = {0};
+  char dbStr[TSDB_DB_NAME_LEN] = {0};
+  char userStr[TSDB_USER_LEN]  = {0};
+  char passStr[TSDB_PASSWORD_LEN]   = {0};
+
+  strncpy(ipStr,   ip,   MIN(TSDB_EP_LEN - 1, ipLen));
+  strncpy(userStr, user, MIN(TSDB_USER_LEN - 1, userLen));
+  strncpy(passStr, pass, MIN(TSDB_PASSWORD_LEN - 1, passLen));
+  strncpy(dbStr,   db,   MIN(TSDB_DB_NAME_LEN - 1, dbLen));
+  return taos_connect(ipStr, userStr, passStr, dbStr, port);
+}
 

source/client/src/clientImpl.c

+#include <tpagedfile.h>
+#include "clientInt.h"
+#include "tdef.h"
+#include "tep.h"
+#include "tglobal.h"
+#include "tmsgtype.h"
+#include "tref.h"
+#include "tscLog.h"
+
+static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet);
+static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody);
+static void destroyConnectMsg(SRequestMsgBody* pMsgBody);
+
+static int32_t sendMsgToServer(void *pTransporter, SEpSet* epSet, const SRequestMsgBody *pBody, int64_t* pTransporterId);
+
+static bool stringLengthCheck(const char* str, size_t maxsize) {
+  if (str == NULL) {
+    return false;
+  }
+
+  size_t len = strlen(str);
+  if (len <= 0 || len > maxsize) {
+    return false;
+  }
+
+  return true;
+}
+
+static bool validateUserName(const char* user) {
+  return stringLengthCheck(user, TSDB_USER_LEN - 1);
+}
+
+static bool validatePassword(const char* passwd) {
+  return stringLengthCheck(passwd, TSDB_PASSWORD_LEN - 1);
+}
+
+static bool validateDbName(const char* db) {
+  return stringLengthCheck(db, TSDB_DB_NAME_LEN - 1);
+}
+
+static char* getClusterKey(const char* user, const char* auth, const char* ip, int32_t port) {
+  char key[512] = {0};
+  snprintf(key, sizeof(key), "%s:%s:%s:%d", user, auth, ip, port);
+  return strdup(key);
+}
+
+static STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo);
+
+TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) {
+  if (taos_init() != TSDB_CODE_SUCCESS) {
+    return NULL;
+  }
+
+  if (!validateUserName(user)) {
+    terrno = TSDB_CODE_TSC_INVALID_USER_LENGTH;
+    return NULL;
+  }
+
+  char tmp[TSDB_DB_NAME_LEN] = {0};
+  if (db != NULL) {
+    if(!validateDbName(db)) {
+      terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH;
+      return NULL;
+    }
+
+    tstrncpy(tmp, db, sizeof(tmp));
+    strdequote(tmp);
+  }
+
+  char secretEncrypt[32] = {0};
+  if (auth == NULL) {
+    if (!validatePassword(pass)) {
+      terrno = TSDB_CODE_TSC_INVALID_PASS_LENGTH;
+      return NULL;
+    }
+
+    taosEncryptPass_c((uint8_t *)pass, strlen(pass), secretEncrypt);
+  } else {
+    tstrncpy(secretEncrypt, auth, tListLen(secretEncrypt));
+  }
+
+  SCorEpSet epSet = {0};
+  if (ip) {
+    if (initEpSetFromCfg(ip, NULL, &epSet) < 0) {
+      return NULL;
+    }
+
+    if (port) {
+      epSet.epSet.port[0] = port;
+    }
+  } else {
+    if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) {
+      return NULL;
+    }
+  }
+
+  char* key = getClusterKey(user, secretEncrypt, ip, port);
+
+  SAppInstInfo* pInst = taosHashGet(appInfo.pInstMap, key, strlen(key));
+  if (pInst == NULL) {
+    pInst = calloc(1, sizeof(struct SAppInstInfo));
+
+    pInst->mgmtEp       = epSet;
+    pInst->pTransporter = openTransporter(user, secretEncrypt);
+
+    taosHashPut(appInfo.pInstMap, key, strlen(key), &pInst, POINTER_BYTES);
+  }
+
+  return taosConnectImpl(ip, user, &secretEncrypt[0], db, port, NULL, NULL, pInst);
+}
+
+int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet) {
+  pEpSet->version = 0;
+
+  // init mgmt ip set
+  SEpSet *mgmtEpSet   = &(pEpSet->epSet);
+  mgmtEpSet->numOfEps = 0;
+  mgmtEpSet->inUse    = 0;
+
+  if (firstEp && firstEp[0] != 0) {
+    if (strlen(firstEp) >= TSDB_EP_LEN) {
+      terrno = TSDB_CODE_TSC_INVALID_FQDN;
+      return -1;
+    }
+
+    taosGetFqdnPortFromEp(firstEp, mgmtEpSet->fqdn[0], &(mgmtEpSet->port[0]));
+    mgmtEpSet->numOfEps++;
+  }
+
+  if (secondEp && secondEp[0] != 0) {
+    if (strlen(secondEp) >= TSDB_EP_LEN) {
+      terrno = TSDB_CODE_TSC_INVALID_FQDN;
+      return -1;
+    }
+
+    taosGetFqdnPortFromEp(secondEp, mgmtEpSet->fqdn[mgmtEpSet->numOfEps], &(mgmtEpSet->port[mgmtEpSet->numOfEps]));
+    mgmtEpSet->numOfEps++;
+  }
+
+  if (mgmtEpSet->numOfEps == 0) {
+    terrno = TSDB_CODE_TSC_INVALID_FQDN;
+    return -1;
+  }
+
+  return 0;
+}
+
+STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo) {
+  STscObj *pTscObj = createTscObj(user, auth, ip, port, pAppInfo);
+  if (NULL == pTscObj) {
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    return pTscObj;
+  }
+
+  SRequestObj *pRequest = createRequest(pTscObj, fp, param, TSDB_SQL_CONNECT);
+  if (pRequest == NULL) {
+    destroyTscObj(pTscObj);
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  SRequestMsgBody body = {0};
+  buildConnectMsg(pRequest, &body);
+
+  int64_t transporterId = 0;
+  sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId);
+
+  tsem_wait(&pRequest->body.rspSem);
+  destroyConnectMsg(&body);
+
+  if (pRequest->code != TSDB_CODE_SUCCESS) {
+    const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(terrno);
+    printf("failed to connect to server, reason: %s\n\n", errorMsg);
+
+    destroyRequest(pRequest);
+    taos_close(pTscObj);
+    pTscObj = NULL;
+  } else {
+    tscDebug("0x%"PRIx64" connection is opening, connId:%d, dnodeConn:%p", pTscObj->id, pTscObj->connId, pTscObj->pTransporter);
+    destroyRequest(pRequest);
+  }
+
+  return pTscObj;
+}
+
+static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) {
+  pMsgBody->msgType         = TSDB_MSG_TYPE_CONNECT;
+  pMsgBody->msgLen          = sizeof(SConnectMsg);
+  pMsgBody->requestObjRefId = pRequest->self;
+
+  SConnectMsg *pConnect = calloc(1, sizeof(SConnectMsg));
+  if (pConnect == NULL) {
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  // TODO refactor full_name
+  char *db;  // ugly code to move the space
+
+  STscObj *pObj = pRequest->pTscObj;
+  pthread_mutex_lock(&pObj->mutex);
+  db = strstr(pObj->db, TS_PATH_DELIMITER);
+
+  db = (db == NULL) ? pObj->db : db + 1;
+  tstrncpy(pConnect->db, db, sizeof(pConnect->db));
+  pthread_mutex_unlock(&pObj->mutex);
+
+  pConnect->pid = htonl(appInfo.pid);
+  pConnect->startTime = htobe64(appInfo.startTime);
+  tstrncpy(pConnect->app, appInfo.appName, tListLen(pConnect->app));
+
+  pMsgBody->pData = pConnect;
+  return 0;
+}
+
+static void destroyConnectMsg(SRequestMsgBody* pMsgBody) {
+  assert(pMsgBody != NULL);
+  tfree(pMsgBody->pData);
+}
+
+int32_t sendMsgToServer(void *pTransporter, SEpSet* epSet, const SRequestMsgBody *pBody, int64_t* pTransporterId) {
+  char *pMsg = rpcMallocCont(pBody->msgLen);
+  if (NULL == pMsg) {
+    tscError("0x%"PRIx64" msg:%s malloc failed", pBody->requestId, taosMsg[pBody->msgType]);
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  memcpy(pMsg, pBody->pData, pBody->msgLen);
+  SRpcMsg rpcMsg = {
+      .msgType = pBody->msgType,
+      .pCont   = pMsg,
+      .contLen = pBody->msgLen,
+      .ahandle = (void*) pBody->requestObjRefId,
+      .handle  = NULL,
+      .code    = 0
+  };
+
+  rpcSendRequest(pTransporter, epSet, &rpcMsg, pTransporterId);
+  return TSDB_CODE_SUCCESS;
+}
+
+void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
+  int64_t requestRefId = (int64_t)pMsg->ahandle;
+
+  SRequestObj *pRequest = (SRequestObj *)taosAcquireRef(tscReqRef, requestRefId);
+  if (pRequest == NULL) {
+    rpcFreeCont(pMsg->pCont);
+    return;
+  }
+
+  assert(pRequest->self == requestRefId);
+  pRequest->metric.rsp = taosGetTimestampMs();
+
+  pRequest->code = pMsg->code;
+
+  STscObj *pTscObj = pRequest->pTscObj;
+  if (pEpSet) {
+    if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, pEpSet)) {
+      updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, pEpSet);
+    }
+  }
+
+  /*
+   * There is not response callback function for submit response.
+   * The actual inserted number of points is the first number.
+   */
+  if (pMsg->code == TSDB_CODE_SUCCESS) {
+    tscDebug("0x%" PRIx64 " message:%s, code:%s rspLen:%d, elapsed:%"PRId64 " ms", pRequest->requestId, taosMsg[pMsg->msgType],
+             tstrerror(pMsg->code), pMsg->contLen, pRequest->metric.rsp - pRequest->metric.start);
+    if (handleRequestRspFp[pRequest->type]) {
+      pMsg->code = (*handleRequestRspFp[pRequest->type])(pRequest, pMsg->pCont, pMsg->contLen);
+    }
+  } else {
+    tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d", pRequest->requestId, taosMsg[pMsg->msgType],
+             tstrerror(pMsg->code), pMsg->contLen);
+  }
+
+  taosReleaseRef(tscReqRef, requestRefId);
+  rpcFreeCont(pMsg->pCont);
+
+  sem_post(&pRequest->body.rspSem);
+}

source/client/src/clientmain.c

+#include "clientInt.h"
+#include "trpc.h"
+#include "os.h"
+#include "taosmsg.h"
+#include "tcache.h"
+#include "tconfig.h"
+#include "tglobal.h"
+#include "tnote.h"
+#include "tref.h"
+#include "tscLog.h"
+#include "tsched.h"
+#include "ttime.h"
+#include "ttimezone.h"
+
+#define TSC_VAR_NOT_RELEASE 1
+#define TSC_VAR_RELEASED    0
+
+static int32_t sentinel = TSC_VAR_NOT_RELEASE;
+static pthread_once_t tscinit = PTHREAD_ONCE_INIT;
+
+extern int32_t tscInitRes;
+
+int taos_options(TSDB_OPTION option, const void *arg, ...) {
+  static int32_t lock = 0;
+
+  for (int i = 1; atomic_val_compare_exchange_32(&lock, 0, 1) != 0; ++i) {
+    if (i % 1000 == 0) {
+      tscInfo("haven't acquire lock after spin %d times.", i);
+      sched_yield();
+    }
+  }
+
+  int ret = taos_options_imp(option, (const char*)arg);
+
+  atomic_store_32(&lock, 0);
+  return ret;
+}
+
+int taos_init() {
+  pthread_once(&tscinit, taos_init_imp);
+  return tscInitRes;
+}
+
+// this function may be called by user or system, or by both simultaneously.
+void taos_cleanup(void) {
+  tscDebug("start to cleanup client environment");
+
+  if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) {
+    return;
+  }
+
+  int32_t id = tscReqRef;
+  tscReqRef = -1;
+  taosCloseRef(id);
+
+  void* p = tscQhandle;
+  tscQhandle = NULL;
+  taosCleanUpScheduler(p);
+
+  id = tscConnRef;
+  tscConnRef = -1;
+  taosCloseRef(id);
+
+  rpcCleanup();
+  taosCloseLog();
+}
+
+void taos_close(TAOS* taos) {
+  if (taos == NULL) {
+    return;
+  }
+
+  STscObj *pTscObj = (STscObj *)taos;
+  tscDebug("0x%"PRIx64" try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs);
+
+  taosRemoveRef(tscConnRef, pTscObj->id);
+}
+
+const char *taos_errstr(TAOS_RES *res) {
+
+}
+
+void taos_free_result(TAOS_RES *res) {
+
+}
\ No newline at end of file

source/client/test/CMakeLists.txt

+
+MESSAGE(STATUS "build parser unit test")
+
+# GoogleTest requires at least C++11
+SET(CMAKE_CXX_STANDARD 11)
+AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+
+ADD_EXECUTABLE(clientTest ${SOURCE_LIST})
+TARGET_LINK_LIBRARIES(
+        clientTest
+        PUBLIC os util common transport gtest taos
+)
+
+TARGET_INCLUDE_DIRECTORIES(
+        clientTest
+        PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/client/"
+        PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/client/inc"
+)

source/client/test/clientTests.cpp

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <gtest/gtest.h>
+#include <iostream>
+#include "tglobal.h"
+#pragma GCC diagnostic ignored "-Wwrite-strings"
+
+#pragma GCC diagnostic ignored "-Wunused-function"
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#pragma GCC diagnostic ignored "-Wsign-compare"
+
+#include "taos.h"
+
+namespace {
+}  // namespace
+
+int main(int argc, char** argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+TEST(testCase, driverInit_Test) {
+  TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
+  assert(pConn != NULL);
+
+  taos_close(pConn);
+}
\ No newline at end of file

source/common/inc/commonInt.h

@@ -24,12 +24,11 @@ extern "C" {
 #include "tlog.h"
 
 extern int32_t cDebugFlag;
-extern int8_t  tscEmbedded;
 
-#define tscFatal(...)  do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0)
-#define tscError(...)  do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0)
-#define tscWarn(...)   do { if (cDebugFlag & DEBUG_WARN)  { taosPrintLog("TSC WARN ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }}  while(0)
-#define tscInfo(...)   do { if (cDebugFlag & DEBUG_INFO)  { taosPrintLog("TSC ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0)
+#define tscFatal(...)  do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", cDebugFlag, __VA_ARGS__); }} while(0)
+#define tscError(...)  do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", cDebugFlag, __VA_ARGS__); }} while(0)
+#define tscWarn(...)   do { if (cDebugFlag & DEBUG_WARN)  { taosPrintLog("TSC WARN ", cDebugFlag, __VA_ARGS__); }}  while(0)
+#define tscInfo(...)   do { if (cDebugFlag & DEBUG_INFO)  { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0)
 #define tscDebug(...)  do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0)
 #define tscTrace(...)  do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", cDebugFlag, __VA_ARGS__); }} while(0)
 #define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", cDebugFlag, __VA_ARGS__); }} while(0)

source/common/src/tvariant.c

@@ -75,7 +75,7 @@ int32_t toInteger(const char* z, int32_t n, int32_t base, int64_t* value, bool*
   return 0;
 }
 
-void taosVariantCreate(SVariant *pVar, char* z, int32_t n, int32_t type) {
+void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type) {
   int32_t ret = 0;
   memset(pVar, 0, sizeof(SVariant));
 

source/dnode/mgmt/daemon/src/daemon.c

@@ -112,12 +112,12 @@ int dmnReadConfig(const char *path) {
     return -1;
   }
 
-  if (taosReadGlobalCfg() != 0) {
+  if (taosReadCfgFromFile() != 0) {
     uError("failed to read global config");
     return -1;
   }
 
-  if (taosCheckGlobalCfg() != 0) {
+  if (taosCheckAndPrintCfg() != 0) {
     uError("failed to check global config");
     return -1;
   }
@@ -142,7 +142,6 @@ void dmnInitOption(SDnodeOpt *pOption) {
   pOption->numOfSupportVnodes = 1;
   pOption->numOfSupportQnodes = 1;
   pOption->statusInterval = tsStatusInterval;
-  pOption->mnodeEqualVnodeNum = tsMnodeEqualVnodeNum;
   pOption->numOfThreadsPerCore = tsNumOfThreadsPerCore;
   pOption->ratioOfQueryCores = tsRatioOfQueryCores;
   pOption->maxShellConns = tsMaxShellConns;

source/dnode/mgmt/impl/inc/dndDnode.h

@@ -31,6 +31,7 @@ int32_t dndGetClusterId(SDnode *pDnode);
 void    dndGetDnodeEp(SDnode *pDnode, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort);
 void    dndGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet);
 void    dndSendRedirectMsg(SDnode *pDnode, SRpcMsg *pMsg);
+void    dndSendStatusMsg(SDnode *pDnode);
 
 #ifdef __cplusplus
 }