Merge remote-tracking branch 'origin/3.0' into feature/3.0_liaohj

.clang-format

@@ -5,6 +5,7 @@ AccessModifierOffset: -1
 AlignAfterOpenBracket: Align
 AlignConsecutiveAssignments: false
 AlignConsecutiveDeclarations: true
+AlignConsecutiveMacros: true
 AlignEscapedNewlinesLeft: true
 AlignOperands:   true
 AlignTrailingComments: true
@@ -86,6 +87,5 @@ SpacesInSquareBrackets: false
 Standard:        Auto
 TabWidth:        8
 UseTab:          Never
-AlignConsecutiveDeclarations: true
 ...
 

.gitmodules

@@ -14,3 +14,6 @@
 	path = tests
 	url = https://github.com/taosdata/tests
 	branch = 3.0
+[submodule "examples/rust"]
+	path = examples/rust
+	url = https://github.com/songtianyi/tdengine-rust-bindings.git

cmake/bdb_CMakeLists.txt.in

@@ -10,4 +10,4 @@ ExternalProject_Add(bdb
         BUILD_COMMAND "$(MAKE)"
         INSTALL_COMMAND ""
         TEST_COMMAND ""
-)
+)
\ No newline at end of file

cmake/cmake.options

@@ -47,13 +47,13 @@ option(
 option(
     BUILD_WITH_UV
     "If build with libuv" 
-    OFF
+    ON  
 )
 
 option(
     BUILD_WITH_UV_TRANS
     "If build with libuv_trans " 
-    OFF 
+    ON 
 )
 
 option(

contrib/CMakeLists.txt

@@ -193,6 +193,7 @@ endif(${BUILD_WITH_TRAFT})
 
 # LIBUV
 if(${BUILD_WITH_UV})
+    add_compile_options(-Wno-sign-compare)
     add_subdirectory(libuv)
 endif(${BUILD_WITH_UV})
 

contrib/test/bdb/CMakeLists.txt

@@ -4,4 +4,4 @@ target_sources(
     "bdbTest.c"
 )
 
-target_link_libraries(bdbTest bdb)
+target_link_libraries(bdbTest bdb)
\ No newline at end of file

contrib/test/bdb/bdbTest.c

@@ -185,4 +185,4 @@ static int idx_callback(DB *       sdbp,   /* secondary db handle */
   skeyp->data = tmpdbt;
 
   return 0;
-}
+}
\ No newline at end of file

example/src/tmq.c

@@ -28,7 +28,7 @@ int32_t init_env() {
     return -1;
   }
 
-  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2");
+  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1");
   if (taos_errno(pRes) != 0) {
     printf("error in create db, reason:%s\n", taos_errstr(pRes));
     return -1;
@@ -56,14 +56,31 @@ int32_t init_env() {
   }
   taos_free_result(pRes);
 
-//  pRes = taos_query(pConn, "create table if not exists tu6 using st1 tags(2)");
-//  if (taos_errno(pRes) != 0) {
-//    printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes));
-//    return -1;
-//  }
-//  taos_free_result(pRes);
+  pRes = taos_query(pConn, "create table if not exists tu2 using st1 tags(2)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+  return 0;
+}
 
-  const char* sql = "select * from st1";
+int32_t create_topic() {
+  printf("create topic\n");
+  TAOS_RES* pRes;
+  TAOS*     pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  if (pConn == NULL) {
+    return -1;
+  }
+
+  pRes = taos_query(pConn, "use abc1");
+  if (taos_errno(pRes) != 0) {
+    printf("error in use db, reason:%s\n", taos_errstr(pRes));
+    return -1;
+  }
+  taos_free_result(pRes);
+
+  const char* sql = "select * from tu1";
   pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));
   if (taos_errno(pRes) != 0) {
     printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes));
@@ -74,6 +91,10 @@ int32_t init_env() {
   return 0;
 }
 
+void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) {
+  printf("commit %d\n", resp);
+}
+
 tmq_t* build_consumer() {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   assert(pConn != NULL);
@@ -86,13 +107,9 @@ tmq_t* build_consumer() {
 
   tmq_conf_t* conf = tmq_conf_new();
   tmq_conf_set(conf, "group.id", "tg2");
+  tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print);
   tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0);
   return tmq;
-
-  tmq_list_t* topic_list = tmq_list_new();
-  tmq_list_append(topic_list, "test_stb_topic_1");
-  tmq_subscribe(tmq, topic_list);
-  return NULL;
 }
 
 tmq_list_t* build_topic_list() {
@@ -109,12 +126,12 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
     printf("subscribe err\n");
     return;
   }
-  int32_t cnt = 0;
+  /*int32_t cnt = 0;*/
   /*clock_t startTime = clock();*/
   while (running) {
     tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500);
     if (tmqmessage) {
-      cnt++;
+      /*cnt++;*/
       msg_process(tmqmessage);
       tmq_message_destroy(tmqmessage);
       /*} else {*/
@@ -132,7 +149,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
 }
 
 void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
-  static const int MIN_COMMIT_COUNT = 1000;
+  static const int MIN_COMMIT_COUNT = 1;
 
   int            msg_count = 0;
   tmq_resp_err_t err;
@@ -192,12 +209,16 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics) {
     fprintf(stderr, "%% Consumer closed\n");
 }
 
-int main() {
+int main(int argc, char* argv[]) {
   int code;
-  code = init_env();
+  if (argc > 1) {
+    printf("env init\n");
+    code = init_env();
+  }
+  create_topic();
   tmq_t*      tmq = build_consumer();
   tmq_list_t* topic_list = build_topic_list();
-//  perf_loop(tmq, topic_list);
-  basic_consume_loop(tmq, topic_list);
-  /*sync_consume_loop(tmq, topic_list);*/
+  /*perf_loop(tmq, topic_list);*/
+  /*basic_consume_loop(tmq, topic_list);*/
+  sync_consume_loop(tmq, topic_list);
 }

rust @ 1c8924dc

+Subproject commit 1c8924dc668e6aa848214c2fc54e3ace3f5bf8df

include/client/taos.h

@@ -31,26 +31,26 @@ typedef void   TAOS_SUB;
 typedef void **TAOS_ROW;
 
 // Data type definition
-#define TSDB_DATA_TYPE_NULL 0        // 1 bytes
-#define TSDB_DATA_TYPE_BOOL 1        // 1 bytes
-#define TSDB_DATA_TYPE_TINYINT 2     // 1 byte
-#define TSDB_DATA_TYPE_SMALLINT 3    // 2 bytes
-#define TSDB_DATA_TYPE_INT 4         // 4 bytes
-#define TSDB_DATA_TYPE_BIGINT 5      // 8 bytes
-#define TSDB_DATA_TYPE_FLOAT 6       // 4 bytes
-#define TSDB_DATA_TYPE_DOUBLE 7      // 8 bytes
-#define TSDB_DATA_TYPE_BINARY 8      // string, alias for varchar
+#define TSDB_DATA_TYPE_NULL      0   // 1 bytes
+#define TSDB_DATA_TYPE_BOOL      1   // 1 bytes
+#define TSDB_DATA_TYPE_TINYINT   2   // 1 byte
+#define TSDB_DATA_TYPE_SMALLINT  3   // 2 bytes
+#define TSDB_DATA_TYPE_INT       4   // 4 bytes
+#define TSDB_DATA_TYPE_BIGINT    5   // 8 bytes
+#define TSDB_DATA_TYPE_FLOAT     6   // 4 bytes
+#define TSDB_DATA_TYPE_DOUBLE    7   // 8 bytes
+#define TSDB_DATA_TYPE_BINARY    8   // string, alias for varchar
 #define TSDB_DATA_TYPE_TIMESTAMP 9   // 8 bytes
-#define TSDB_DATA_TYPE_NCHAR 10      // unicode string
-#define TSDB_DATA_TYPE_UTINYINT 11   // 1 byte
+#define TSDB_DATA_TYPE_NCHAR     10  // unicode string
+#define TSDB_DATA_TYPE_UTINYINT  11  // 1 byte
 #define TSDB_DATA_TYPE_USMALLINT 12  // 2 bytes
-#define TSDB_DATA_TYPE_UINT 13       // 4 bytes
-#define TSDB_DATA_TYPE_UBIGINT 14    // 8 bytes
-#define TSDB_DATA_TYPE_VARCHAR 15    // string
+#define TSDB_DATA_TYPE_UINT      13  // 4 bytes
+#define TSDB_DATA_TYPE_UBIGINT   14  // 8 bytes
+#define TSDB_DATA_TYPE_VARCHAR   15  // string
 #define TSDB_DATA_TYPE_VARBINARY 16  // binary
-#define TSDB_DATA_TYPE_JSON 17       // json
-#define TSDB_DATA_TYPE_DECIMAL 18    // decimal
-#define TSDB_DATA_TYPE_BLOB 19       // binary
+#define TSDB_DATA_TYPE_JSON      17  // json
+#define TSDB_DATA_TYPE_DECIMAL   18  // decimal
+#define TSDB_DATA_TYPE_BLOB      19  // binary
 
 typedef enum {
   TSDB_OPTION_LOCALE,
@@ -198,8 +198,8 @@ DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLi
 /* --------------------------TMQ INTERFACE------------------------------- */
 
 enum tmq_resp_err_t {
+  TMQ_RESP_ERR__FAIL = -1,
   TMQ_RESP_ERR__SUCCESS = 0,
-  TMQ_RESP_ERR__FAIL = 1,
 };
 
 typedef enum tmq_resp_err_t tmq_resp_err_t;
@@ -226,7 +226,7 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
 DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list);
 #if 0
 DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq);
-DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_topic_vgroup_list_t** topics);
+DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics);
 #endif
 DLL_EXPORT tmq_message_t *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time);
 DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq);
@@ -238,6 +238,7 @@ DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *
 #if 0
 DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async);
 #endif
+DLL_EXPORT tmq_resp_err_t tmq_seek(tmq_t *tmq, const tmq_topic_vgroup_t *offset);
 /* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */
 
 enum tmq_conf_res_t {

include/common/common.h

@@ -16,7 +16,6 @@
 #ifndef TDENGINE_COMMON_H
 #define TDENGINE_COMMON_H
 
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -43,14 +42,16 @@ extern "C" {
 //   int16_t bytes;
 // } SSchema;
 
-#define TMQ_REQ_TYPE_COMMIT_ONLY 0
-#define TMQ_REQ_TYPE_CONSUME_ONLY 1
-#define TMQ_REQ_TYPE_CONSUME_AND_COMMIT 2
+enum {
+  TMQ_CONF__RESET_OFFSET__LATEST = -1,
+  TMQ_CONF__RESET_OFFSET__EARLIEAST = -2,
+  TMQ_CONF__RESET_OFFSET__NONE = -3,
+};
 
 typedef struct {
   uint32_t  numOfTables;
-  SArray   *pGroupList;
-  SHashObj *map;  // speedup acquire the tableQueryInfo by table uid
+  SArray*   pGroupList;
+  SHashObj* map;  // speedup acquire the tableQueryInfo by table uid
 } STableGroupInfo;
 
 typedef struct SColumnDataAgg {
@@ -79,14 +80,14 @@ typedef struct SConstantItem {
 
 // info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList);
 typedef struct SSDataBlock {
-  SColumnDataAgg *pBlockAgg;
-  SArray         *pDataBlock;    // SArray<SColumnInfoData>
-  SArray         *pConstantList; // SArray<SConstantItem>, it is a constant/tags value of the corresponding result value.
-  SDataBlockInfo  info;
+  SColumnDataAgg* pBlockAgg;
+  SArray*         pDataBlock;  // SArray<SColumnInfoData>
+  SArray* pConstantList;       // SArray<SConstantItem>, it is a constant/tags value of the corresponding result value.
+  SDataBlockInfo info;
 } SSDataBlock;
 
 typedef struct SVarColAttr {
-  int32_t *offset;    // start position for each entry in the list
+  int32_t* offset;    // start position for each entry in the list
   uint32_t length;    // used buffer size that contain the valid data
   uint32_t allocLen;  // allocated buffer size
 } SVarColAttr;
@@ -94,11 +95,11 @@ typedef struct SVarColAttr {
 // pBlockAgg->numOfNull == info.rows, all data are null
 // pBlockAgg->numOfNull == 0, no data are null.
 typedef struct SColumnInfoData {
-  SColumnInfo info;   // TODO filter info needs to be removed
-  bool        hasNull;// if current column data has null value.
-  char       *pData;  // the corresponding block data in memory
+  SColumnInfo info;     // TODO filter info needs to be removed
+  bool        hasNull;  // if current column data has null value.
+  char*       pData;    // the corresponding block data in memory
   union {
-    char       *nullbitmap;  // bitmap, one bit for each item in the list
+    char*       nullbitmap;  // bitmap, one bit for each item in the list
     SVarColAttr varmeta;
   };
 } SColumnInfoData;
@@ -149,7 +150,6 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp
   int32_t tlen = 0;
   int32_t sz = 0;
   tlen += taosEncodeFixedI64(buf, pRsp->consumerId);
-  tlen += taosEncodeFixedI64(buf, pRsp->committedOffset);
   tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
   tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
   tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum);
@@ -170,7 +170,6 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp
 static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) {
   int32_t sz;
   buf = taosDecodeFixedI64(buf, &pRsp->consumerId);
-  buf = taosDecodeFixedI64(buf, &pRsp->committedOffset);
   buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
   buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
   buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);
@@ -250,11 +249,11 @@ typedef struct SSqlExpr {
   char    token[TSDB_COL_NAME_LEN];  // original token
   SSchema resSchema;
 
-  int32_t   numOfCols;
-  SColumn*  pColumns;       // data columns that are required by query
-  int32_t   interBytes;     // inter result buffer size
-  int16_t   numOfParams;    // argument value of each function
-  SVariant  param[3];       // parameters are not more than 3
+  int32_t  numOfCols;
+  SColumn* pColumns;     // data columns that are required by query
+  int32_t  interBytes;   // inter result buffer size
+  int16_t  numOfParams;  // argument value of each function
+  SVariant param[3];     // parameters are not more than 3
 } SSqlExpr;
 
 typedef struct SExprInfo {
@@ -271,7 +270,7 @@ typedef struct SSessionWindow {
   SColumn col;
 } SSessionWindow;
 
-#define QUERY_ASC_FORWARD_STEP 1
+#define QUERY_ASC_FORWARD_STEP  1
 #define QUERY_DESC_FORWARD_STEP -1
 
 #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP)

include/common/tcompare.h

@@ -23,8 +23,12 @@
 extern "C" {
 #endif
 
-int32_t       compareStrPatternComp(const void* pLeft, const void* pRight);
-int32_t       compareWStrPatternComp(const void* pLeft, const void* pRight);
+int32_t compareStrPatternMatch(const void* pLeft, const void* pRight);
+int32_t compareStrPatternNotMatch(const void* pLeft, const void* pRight);
+
+int32_t compareWStrPatternMatch(const void* pLeft, const void* pRight);
+int32_t compareWStrPatternNotMatch(const void* pLeft, const void* pRight);
+
 __compar_fn_t getComparFunc(int32_t type, int32_t optr);
 __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
 int32_t       doCompare(const char* a, const char* b, int32_t type, size_t size);
@@ -33,4 +37,4 @@ int32_t       doCompare(const char* a, const char* b, int32_t type, size_t size)
 }
 #endif
 
-#endif /*_TD_TCOMPARE_H_*/
+#endif /*_TD_TCOMPARE_H_*/
\ No newline at end of file

include/common/tep.h

@@ -20,7 +20,7 @@ typedef struct SBlockOrderInfo {
   SColumnInfoData *pColData;
 } SBlockOrderInfo;
 
-int  taosGetFqdnPortFromEp(const char *ep, SEp *pEp);
+int  taosGetFqdnPortFromEp(const char *ep, uint16_t defaultPort, SEp *pEp);
 void addEpIntoEpSet(SEpSet *pEpSet, const char *fqdn, uint16_t port);
 
 bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2);

include/common/tglobal.h

@@ -21,35 +21,11 @@ extern "C" {
 #endif
 
 #include "tdef.h"
-#include "tcfg.h"
-
-// cluster
-extern char     tsFirst[];
-extern char     tsSecond[];
-extern char     tsLocalFqdn[];
-extern char     tsLocalEp[];
-extern uint16_t tsServerPort;
-extern int32_t  tsStatusInterval;
-extern int8_t   tsEnableTelemetryReporting;
-extern int32_t  tsNumOfSupportVnodes;
 
 // common
-extern int      tsRpcTimer;
-extern int      tsRpcMaxTime;
-extern int      tsRpcForceTcp;  // all commands go to tcp protocol if this is enabled
-extern int32_t  tsMaxConnections;
-extern int32_t  tsMaxShellConns;
-extern int32_t  tsShellActivityTimer;
-extern uint32_t tsMaxTmrCtrl;
-extern float    tsNumOfThreadsPerCore;
-extern int32_t  tsNumOfCommitThreads;
-extern float    tsRatioOfQueryCores;
-extern int8_t   tsDaylight;
-extern int8_t   tsEnableCoreFile;
 extern int32_t  tsCompressMsgSize;
 extern int32_t  tsCompressColData;
 extern int32_t  tsMaxNumOfDistinctResults;
-extern char     tsTempDir[];
 extern int      tsCompatibleModel;  // 2.0 compatible model
 extern int8_t   tsEnableSlaveQuery;
 extern int8_t   tsEnableAdjustMaster;
@@ -66,7 +42,6 @@ extern int8_t  tsDeadLockKillQuery;
 // client
 extern int32_t tsMaxWildCardsLen;
 extern int32_t tsMaxRegexStringLen;
-extern int8_t  tsTscEnableRecordSql;
 extern int32_t tsMaxNumOfOrderedResults;
 extern int32_t tsMinSlidingTime;
 extern int32_t tsMinIntervalTime;
@@ -77,41 +52,8 @@ extern float   tsStreamComputDelayRatio;  // the delayed computing ration of the
 extern int32_t tsProjectExecInterval;
 extern int64_t tsMaxRetentWindow;
 
-// system info
-extern float    tsTotalLogDirGB;
-extern float    tsTotalTmpDirGB;
-extern float    tsTotalDataDirGB;
-extern float    tsAvailLogDirGB;
-extern float    tsAvailTmpDirectorySpace;
-extern float    tsAvailDataDirGB;
-extern float    tsUsedDataDirGB;
-extern float    tsMinimalLogDirGB;
-extern float    tsReservedTmpDirectorySpace;
-extern float    tsMinimalDataDirGB;
-extern uint32_t tsVersion;
-
-// build info
-extern char version[];
-extern char compatible_version[];
-extern char gitinfo[];
-extern char gitinfoOfInternal[];
-extern char buildinfo[];
-
-// lossy
-extern char     tsLossyColumns[];
-extern double   tsFPrecision;
-extern double   tsDPrecision;
-extern uint32_t tsMaxRange;
-extern uint32_t tsCurRange;
-extern char     tsCompressor[];
-
-extern int32_t  tsDiskCfgNum;
-extern SDiskCfg tsDiskCfg[];
-
 #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
 
-void    taosInitGlobalCfg();
-int32_t taosCheckAndPrintCfg();
 int32_t taosCfgDynamicOptions(char *msg);
 bool    taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId);
 void    taosAddDataDir(int index, char *v1, int level, int primary);

include/common/tmsgdef.h

@@ -13,6 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+// clang-format off
+
 #if 0
 #undef TD_MSG_INFO_
 #undef TD_MSG_NUMBER_
@@ -82,7 +84,6 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_DND_CREATE_VNODE, "dnode-create-vnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE, "dnode-alter-vnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_DROP_VNODE, "dnode-drop-vnode", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_DND_AUTH_VNODE, "dnode-auth-vnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_SYNC_VNODE, "dnode-sync-vnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_COMPACT_VNODE, "dnode-compact-vnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "dnode-config-dnode", NULL, NULL)
@@ -134,16 +135,19 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_SHOW, "mnode-show", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_SHOW_RETRIEVE, "mnode-retrieve", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_STATUS, "mnode-status", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_MND_TRANS, "mnode-trans", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "mnode-trans-tmr", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_KILL_TRANS, "mnode-kill-trans", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_TELEM_TIMER, "mnode-telem-tmr", SMTimerReq, SMTimerReq)
   TD_DEF_MSG_TYPE(TDMT_MND_GRANT, "mnode-grant", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_AUTH, "mnode-auth", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "mnode-create-topic", SCMCreateTopicReq, SCMCreateTopicRsp)
+  TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "mnode-create-topic", SMCreateTopicReq, SMCreateTopicRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp)
-  TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-timer", SMqTmrMsg, SMqTmrMsg)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp)
 
   // Requests handled by VNODE
   TD_NEW_MSG_SEG(TDMT_VND_MSG)

include/common/tname.h

@@ -16,6 +16,11 @@
 #ifndef TDENGINE_TNAME_H
 #define TDENGINE_TNAME_H
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
 #include "tdef.h"
 #include "tmsg.h"
 
@@ -59,4 +64,9 @@ int32_t tNameSetAcctId(SName* dst, int32_t acctId);
 
 SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name);
 
+#ifdef __cplusplus
+}
+#endif
+
+
 #endif  // TDENGINE_TNAME_H

include/common/ttokendef.h

@@ -126,86 +126,87 @@
 #define TK_PRECISION                      108
 #define TK_UPDATE                         109
 #define TK_CACHELAST                      110
-#define TK_UNSIGNED                       111
-#define TK_TAGS                           112
-#define TK_USING                          113
-#define TK_NULL                           114
-#define TK_NOW                            115
-#define TK_SELECT                         116
-#define TK_UNION                          117
-#define TK_ALL                            118
-#define TK_DISTINCT                       119
-#define TK_FROM                           120
-#define TK_VARIABLE                       121
-#define TK_INTERVAL                       122
-#define TK_EVERY                          123
-#define TK_SESSION                        124
-#define TK_STATE_WINDOW                   125
-#define TK_FILL                           126
-#define TK_SLIDING                        127
-#define TK_ORDER                          128
-#define TK_BY                             129
-#define TK_ASC                            130
-#define TK_GROUP                          131
-#define TK_HAVING                         132
-#define TK_LIMIT                          133
-#define TK_OFFSET                         134
-#define TK_SLIMIT                         135
-#define TK_SOFFSET                        136
-#define TK_WHERE                          137
-#define TK_RESET                          138
-#define TK_QUERY                          139
-#define TK_SYNCDB                         140
-#define TK_ADD                            141
-#define TK_COLUMN                         142
-#define TK_MODIFY                         143
-#define TK_TAG                            144
-#define TK_CHANGE                         145
-#define TK_SET                            146
-#define TK_KILL                           147
-#define TK_CONNECTION                     148
-#define TK_STREAM                         149
-#define TK_COLON                          150
-#define TK_ABORT                          151
-#define TK_AFTER                          152
-#define TK_ATTACH                         153
-#define TK_BEFORE                         154
-#define TK_BEGIN                          155
-#define TK_CASCADE                        156
-#define TK_CLUSTER                        157
-#define TK_CONFLICT                       158
-#define TK_COPY                           159
-#define TK_DEFERRED                       160
-#define TK_DELIMITERS                     161
-#define TK_DETACH                         162
-#define TK_EACH                           163
-#define TK_END                            164
-#define TK_EXPLAIN                        165
-#define TK_FAIL                           166
-#define TK_FOR                            167
-#define TK_IGNORE                         168
-#define TK_IMMEDIATE                      169
-#define TK_INITIALLY                      170
-#define TK_INSTEAD                        171
-#define TK_KEY                            172
-#define TK_OF                             173
-#define TK_RAISE                          174
-#define TK_REPLACE                        175
-#define TK_RESTRICT                       176
-#define TK_ROW                            177
-#define TK_STATEMENT                      178
-#define TK_TRIGGER                        179
-#define TK_VIEW                           180
-#define TK_SEMI                           181
-#define TK_NONE                           182
-#define TK_PREV                           183
-#define TK_LINEAR                         184
-#define TK_IMPORT                         185
-#define TK_TBNAME                         186
-#define TK_JOIN                           187
-#define TK_INSERT                         188
-#define TK_INTO                           189
-#define TK_VALUES                         190
+#define TK_STREAM                         111
+#define TK_MODE                           112
+#define TK_UNSIGNED                       113
+#define TK_TAGS                           114
+#define TK_USING                          115
+#define TK_NULL                           116
+#define TK_NOW                            117
+#define TK_SELECT                         118
+#define TK_UNION                          119
+#define TK_ALL                            120
+#define TK_DISTINCT                       121
+#define TK_FROM                           122
+#define TK_VARIABLE                       123
+#define TK_INTERVAL                       124
+#define TK_EVERY                          125
+#define TK_SESSION                        126
+#define TK_STATE_WINDOW                   127
+#define TK_FILL                           128
+#define TK_SLIDING                        129
+#define TK_ORDER                          130
+#define TK_BY                             131
+#define TK_ASC                            132
+#define TK_GROUP                          133
+#define TK_HAVING                         134
+#define TK_LIMIT                          135
+#define TK_OFFSET                         136
+#define TK_SLIMIT                         137
+#define TK_SOFFSET                        138
+#define TK_WHERE                          139
+#define TK_RESET                          140
+#define TK_QUERY                          141
+#define TK_SYNCDB                         142
+#define TK_ADD                            143
+#define TK_COLUMN                         144
+#define TK_MODIFY                         145
+#define TK_TAG                            146
+#define TK_CHANGE                         147
+#define TK_SET                            148
+#define TK_KILL                           149
+#define TK_CONNECTION                     150
+#define TK_COLON                          151
+#define TK_ABORT                          152
+#define TK_AFTER                          153
+#define TK_ATTACH                         154
+#define TK_BEFORE                         155
+#define TK_BEGIN                          156
+#define TK_CASCADE                        157
+#define TK_CLUSTER                        158
+#define TK_CONFLICT                       159
+#define TK_COPY                           160
+#define TK_DEFERRED                       161
+#define TK_DELIMITERS                     162
+#define TK_DETACH                         163
+#define TK_EACH                           164
+#define TK_END                            165
+#define TK_EXPLAIN                        166
+#define TK_FAIL                           167
+#define TK_FOR                            168
+#define TK_IGNORE                         169
+#define TK_IMMEDIATE                      170
+#define TK_INITIALLY                      171
+#define TK_INSTEAD                        172
+#define TK_KEY                            173
+#define TK_OF                             174
+#define TK_RAISE                          175
+#define TK_REPLACE                        176
+#define TK_RESTRICT                       177
+#define TK_ROW                            178
+#define TK_STATEMENT                      179
+#define TK_TRIGGER                        180
+#define TK_VIEW                           181
+#define TK_SEMI                           182
+#define TK_NONE                           183
+#define TK_PREV                           184
+#define TK_LINEAR                         185
+#define TK_IMPORT                         186
+#define TK_TBNAME                         187
+#define TK_JOIN                           188
+#define TK_INSERT                         189
+#define TK_INTO                           190
+#define TK_VALUES                         191
 
 #define NEW_TK_OR                               1
 #define NEW_TK_AND                              2

include/common/ttypes.h

@@ -78,9 +78,12 @@ typedef struct {
       case TSDB_DATA_TYPE_UINT:                      \
         (_v) = (_finalType)GET_UINT32_VAL(_data);    \
         break;                                       \
-      default:                                       \
+      case TSDB_DATA_TYPE_INT:                       \
         (_v) = (_finalType)GET_INT32_VAL(_data);     \
         break;                                       \
+      default:                                       \
+        (_v) = (_finalType)varDataLen(_data);        \
+        break;                                       \
     }                                                \
   } while (0)
 
@@ -88,6 +91,8 @@ typedef struct {
   do {                                         \
     switch (_type) {                           \
       case TSDB_DATA_TYPE_BOOL:                \
+        *(bool *)(_v) = (bool)(_data);     \
+        break;                                 \
       case TSDB_DATA_TYPE_TINYINT:             \
         *(int8_t *)(_v) = (int8_t)(_data);     \
         break;                                 \
@@ -101,6 +106,7 @@ typedef struct {
         *(uint16_t *)(_v) = (uint16_t)(_data); \
         break;                                 \
       case TSDB_DATA_TYPE_BIGINT:              \
+      case TSDB_DATA_TYPE_TIMESTAMP:           \
         *(int64_t *)(_v) = (int64_t)(_data);   \
         break;                                 \
       case TSDB_DATA_TYPE_UBIGINT:             \
@@ -115,9 +121,11 @@ typedef struct {
       case TSDB_DATA_TYPE_UINT:                \
         *(uint32_t *)(_v) = (uint32_t)(_data); \
         break;                                 \
-      default:                                 \
+      case TSDB_DATA_TYPE_INT:                 \
         *(int32_t *)(_v) = (int32_t)(_data);   \
         break;                                 \
+      default:                                 \
+        break;                                 \
     }                                          \
   } while (0)
 
@@ -138,6 +146,9 @@ typedef struct {
 #define IS_VALID_FLOAT(_t)      ((_t) >= -FLT_MAX && (_t) <= FLT_MAX)
 #define IS_VALID_DOUBLE(_t)     ((_t) >= -DBL_MAX && (_t) <= DBL_MAX)
 
+#define IS_CONVERT_AS_SIGNED(_t) (IS_SIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP))
+#define IS_CONVERT_AS_UNSIGNED(_t) (IS_UNSIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL))
+
 static FORCE_INLINE bool isNull(const void *val, int32_t type) {
   switch (type) {
     case TSDB_DATA_TYPE_BOOL:
@@ -205,6 +216,7 @@ void* getDataMax(int32_t type);
 
 
 #define SET_DOUBLE_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_DOUBLE_NULL)
+#define SET_BIGINT_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_BIGINT_NULL)
 
 #ifdef __cplusplus
 }

include/dnode/mgmt/dnode.h

@@ -27,15 +27,18 @@ typedef struct SDnode SDnode;
 
 /* ------------------------ Environment ------------------ */
 typedef struct {
-  int32_t sver;
-  int32_t numOfCores;
-  int16_t numOfCommitThreads;
-  int8_t  enableTelem;
-  char    timezone[TSDB_TIMEZONE_LEN];
-  char    locale[TSDB_LOCALE_LEN];
-  char    charset[TSDB_LOCALE_LEN];
-  char    buildinfo[64];
-  char    gitinfo[48];
+  int32_t  sver;
+  int32_t  numOfCores;
+  uint16_t numOfCommitThreads;
+  bool     enableTelem;
+  bool     printAuth;
+  int32_t  rpcTimer;
+  int32_t  rpcMaxTime;
+  char     timezone[TD_TIMEZONE_LEN];
+  char     locale[TD_LOCALE_LEN];
+  char     charset[TD_LOCALE_LEN];
+  char     buildinfo[64];
+  char     gitinfo[48];
 } SDnodeEnvCfg;
 
 /**
@@ -65,6 +68,7 @@ typedef struct {
   char     localEp[TSDB_EP_LEN];
   char     localFqdn[TSDB_FQDN_LEN];
   char     firstEp[TSDB_EP_LEN];
+  char     secondEp[TSDB_EP_LEN];
 } SDnodeObjCfg;
 
 /**

include/dnode/mnode/mnode.h

@@ -46,7 +46,8 @@ typedef struct SMnodeLoad {
 
 typedef struct SMnodeCfg {
   int32_t sver;
-  int8_t  enableTelem;
+  bool    enableTelem;
+  bool    printAuth;
   int32_t statusInterval;
   int32_t shellActivityTimer;
   char   *timezone;

include/dnode/mnode/sdb/sdb.h

@@ -113,14 +113,15 @@ typedef enum {
   SDB_USER = 7,
   SDB_AUTH = 8,
   SDB_ACCT = 9,
-  SDB_SUBSCRIBE = 10,
-  SDB_CONSUMER = 11,
-  SDB_TOPIC = 12,
-  SDB_VGROUP = 13,
-  SDB_STB = 14,
-  SDB_DB = 15,
-  SDB_FUNC = 16,
-  SDB_MAX = 17
+  SDB_OFFSET = 10,
+  SDB_SUBSCRIBE = 11,
+  SDB_CONSUMER = 12,
+  SDB_TOPIC = 13,
+  SDB_VGROUP = 14,
+  SDB_STB = 15,
+  SDB_DB = 16,
+  SDB_FUNC = 17,
+  SDB_MAX = 18
 } ESdbType;
 
 typedef struct SSdb SSdb;

include/libs/config/config.h

+
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_CONFIG_H_
+#define _TD_CONFIG_H_
+
+#include "os.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define CFG_NAME_MAX_LEN 128
+
+typedef enum {
+  CFG_STYPE_DEFAULT,
+  CFG_STYPE_CFG_FILE,
+  CFG_STYPE_ENV_FILE,
+  CFG_STYPE_ENV_VAR,
+  CFG_STYPE_APOLLO_URL,
+  CFG_STYPE_ARG_LIST,
+  CFG_STYPE_API_OPTION
+} ECfgSrcType;
+
+typedef enum {
+  CFG_DTYPE_NONE,
+  CFG_DTYPE_BOOL,
+  CFG_DTYPE_INT32,
+  CFG_DTYPE_INT64,
+  CFG_DTYPE_FLOAT,
+  CFG_DTYPE_STRING,
+  CFG_DTYPE_IPSTR,
+  CFG_DTYPE_DIR,
+  CFG_DTYPE_LOCALE,
+  CFG_DTYPE_CHARSET,
+  CFG_DTYPE_TIMEZONE
+} ECfgDataType;
+
+typedef struct SConfigItem {
+  ECfgSrcType  stype;
+  ECfgDataType dtype;
+  char        *name;
+  union {
+    bool     bval;
+    float    fval;
+    int32_t  i32;
+    int64_t  i64;
+    char    *str;
+  };
+  union {
+    int64_t imin;
+    double  fmin;
+  };
+  union {
+    int64_t imax;
+    double  fmax;
+  };
+} SConfigItem;
+
+typedef struct SConfig SConfig;
+
+SConfig *cfgInit();
+int32_t  cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr);
+void     cfgCleanup(SConfig *pCfg);
+
+int32_t      cfgGetSize(SConfig *pCfg);
+SConfigItem *cfgIterate(SConfig *pCfg, SConfigItem *pIter);
+void         cfgCancelIterate(SConfig *pCfg, SConfigItem *pIter);
+SConfigItem *cfgGetItem(SConfig *pCfg, const char *name);
+int32_t      cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype);
+
+int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal);
+int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval);
+int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval);
+int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval);
+int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal);
+int32_t cfgAddIpStr(SConfig *pCfg, const char *name, const char *defaultVa);
+int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal);
+int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal);
+int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal);
+int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal);
+
+const char *cfgStypeStr(ECfgSrcType type);
+const char *cfgDtypeStr(ECfgDataType type);
+
+void cfgDumpCfg(SConfig *pCfg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_CONFIG_H_*/

include/libs/function/function.h

@@ -228,13 +228,19 @@ typedef struct SAggFunctionInfo {
   int32_t (*dataReqFunc)(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId);
 } SAggFunctionInfo;
 
-struct SScalarFuncParam;
+typedef struct SScalarParam {
+  void*   data;
+  bool    colData;
+  int32_t num;
+  int32_t type;
+  int32_t bytes;
+} SScalarParam;
 
 typedef struct SScalarFunctionInfo {
   char      name[FUNCTIONS_NAME_MAX_LENGTH];
   int8_t    type;              // scalar function or aggregation function
   uint32_t  functionId;        // index of scalar function
-  void     (*process)(struct SScalarFuncParam* pOutput, size_t numOfInput, const struct SScalarFuncParam *pInput);
+  void     (*process)(struct SScalarParam* pOutput, size_t numOfInput, const struct SScalarParam *pInput);
 } SScalarFunctionInfo;
 
 typedef struct SMultiFunctionsDesc {
@@ -287,10 +293,6 @@ int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num);
 bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry);
 bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry);
 
-struct SScalarFunctionSupport* createScalarFuncSupport(int32_t num);
-void destroyScalarFuncSupport(struct SScalarFunctionSupport* pSupport, int32_t num);
-struct SScalarFunctionSupport* getScalarFuncSupport(struct SScalarFunctionSupport* pSupport, int32_t index);
-
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 // fill api
 struct SFillInfo;

include/libs/function/functionMgt.h

@@ -21,6 +21,7 @@ extern "C" {
 #endif
 
 #include "querynodes.h"
+#include "function.h"
 
 typedef enum EFunctionType {
   // aggregate function
@@ -105,7 +106,6 @@ typedef struct SFuncExecEnv {
   int32_t calcMemSize;
 } SFuncExecEnv;
 
-typedef void* FuncMgtHandle;
 typedef bool (*FExecGetEnv)(SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
 typedef void (*FExecProcess)(struct SqlFunctionCtx *pCtx);
@@ -118,11 +118,16 @@ typedef struct SFuncExecFuncs {
   FExecFinalize finalize;
 } SFuncExecFuncs;
 
-int32_t fmFuncMgtInit();
+typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+
+typedef struct SScalarFuncExecFuncs {
+  FScalarExecProcess process;
+} SScalarFuncExecFuncs;
 
-int32_t fmGetHandle(FuncMgtHandle* pHandle);
 
-int32_t fmGetFuncInfo(FuncMgtHandle handle, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType);
+int32_t fmFuncMgtInit();
+
+int32_t fmGetFuncInfo(const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType);
 
 int32_t fmGetFuncResultType(SFunctionNode* pFunc);
 
@@ -136,7 +141,8 @@ bool fmIsTimeorderFunc(int32_t funcId);
 
 int32_t fmFuncScanType(int32_t funcId);
 
-int32_t fmGetFuncExecFuncs(FuncMgtHandle handle, int32_t funcId, SFuncExecFuncs* pFpSet);
+int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet);
+int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet);
 
 #ifdef __cplusplus
 }

include/libs/nodes/nodes.h

@@ -49,7 +49,6 @@ typedef enum ENodeType {
   QUERY_NODE_VALUE,
   QUERY_NODE_OPERATOR,
   QUERY_NODE_LOGIC_CONDITION,
-  QUERY_NODE_IS_NULL_CONDITION,
   QUERY_NODE_FUNCTION,
   QUERY_NODE_REAL_TABLE,
   QUERY_NODE_TEMP_TABLE,
@@ -62,14 +61,27 @@ typedef enum ENodeType {
   QUERY_NODE_INTERVAL_WINDOW,
   QUERY_NODE_NODE_LIST,
   QUERY_NODE_FILL,
-
-  // Only be used in parser module.
-  QUERY_NODE_RAW_EXPR,
+  QUERY_NODE_RAW_EXPR, // Only be used in parser module.
+  QUERY_NODE_COLUMN_REF,
+  QUERY_NODE_TARGET,
+  QUERY_NODE_TUPLE_DESC,
+  QUERY_NODE_SLOT_DESC,
 
   // Statement nodes are used in parser and planner module.
   QUERY_NODE_SET_OPERATOR,
   QUERY_NODE_SELECT_STMT,
-  QUERY_NODE_SHOW_STMT
+  QUERY_NODE_SHOW_STMT,
+
+  // logic plan node
+  QUERY_NODE_LOGIC_PLAN_SCAN,
+  QUERY_NODE_LOGIC_PLAN_JOIN,
+  QUERY_NODE_LOGIC_PLAN_AGG,
+  QUERY_NODE_LOGIC_PLAN_PROJECT,
+
+  // physical plan node
+  QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN,
+  QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN,
+  QUERY_NODE_PHYSICAL_PLAN_PROJECT
 } ENodeType;
 
 /**
@@ -87,7 +99,7 @@ typedef struct SListCell {
 } SListCell;
 
 typedef struct SNodeList {
-  int16_t length;
+  int32_t length;
   SListCell* pHead;
   SListCell* pTail;
 } SNodeList;
@@ -96,7 +108,8 @@ SNode* nodesMakeNode(ENodeType type);
 void nodesDestroyNode(SNode* pNode);
 
 SNodeList* nodesMakeList();
-SNodeList* nodesListAppend(SNodeList* pList, SNode* pNode);
+int32_t nodesListAppend(SNodeList* pList, SNode* pNode);
+int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc);
 SListCell* nodesListErase(SNodeList* pList, SListCell* pCell);
 SNode* nodesListGetNode(SNodeList* pList, int32_t index);
 void nodesDestroyList(SNodeList* pList);
@@ -121,9 +134,10 @@ void nodesRewriteListPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* p
 
 bool nodesEqualNode(const SNode* a, const SNode* b);
 
-void nodesCloneNode(const SNode* pNode);
+SNode* nodesCloneNode(const SNode* pNode);
+SNodeList* nodesCloneList(const SNodeList* pList);
 
-int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen);
+int32_t nodesNodeToString(const SNode* pNode, bool format, char** pStr, int32_t* pLen);
 int32_t nodesStringToNode(const char* pStr, SNode** pNode);
 
 #ifdef __cplusplus

include/libs/nodes/plannodes.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_PLANN_NODES_H_
+#define _TD_PLANN_NODES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "querynodes.h"
+#include "tmsg.h"
+
+typedef struct SLogicNode {
+  ENodeType type;
+  int32_t id;
+  SNodeList* pTargets; // SColumnNode
+  SNode* pConditions;
+  SNodeList* pChildren;
+  struct SLogicNode* pParent;
+} SLogicNode;
+
+typedef enum EScanType {
+  SCAN_TYPE_TAG,
+  SCAN_TYPE_TABLE,
+  SCAN_TYPE_STABLE,
+  SCAN_TYPE_STREAM
+} EScanType;
+
+typedef struct SScanLogicNode {
+  SLogicNode node;
+  SNodeList* pScanCols;
+  struct STableMeta* pMeta;
+  EScanType scanType;
+  uint8_t scanFlag;         // denotes reversed scan of data or not
+  STimeWindow scanRange;
+} SScanLogicNode;
+
+typedef struct SJoinLogicNode {
+  SLogicNode node;
+  EJoinType joinType;
+  SNode* pOnConditions;
+} SJoinLogicNode;
+
+typedef struct SAggLogicNode {
+  SLogicNode node;
+  SNodeList* pGroupKeys;
+  SNodeList* pAggFuncs;
+} SAggLogicNode;
+
+typedef struct SProjectLogicNode {
+  SLogicNode node;
+  SNodeList* pProjections;
+} SProjectLogicNode;
+
+typedef struct SSlotDescNode {
+  ENodeType type;
+  int16_t slotId;
+  SDataType dataType;
+  int16_t srcTupleId;
+  int16_t srcSlotId;
+  bool reserve;
+  bool output;
+} SSlotDescNode;
+
+typedef struct STupleDescNode {
+  ENodeType type;
+  int16_t tupleId;
+  SNodeList* pSlots;
+} STupleDescNode;
+
+typedef struct SPhysiNode {
+  ENodeType type;
+  STupleDescNode outputTuple;
+  SNode* pConditions;
+  SNodeList* pChildren;
+  struct SPhysiNode* pParent;
+} SPhysiNode;
+
+typedef struct SScanPhysiNode {
+  SPhysiNode  node;
+  SNodeList* pScanCols;
+  uint64_t uid;           // unique id of the table
+  int8_t tableType;
+  int32_t order;         // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC
+  int32_t count;         // repeat count
+  int32_t reverse;       // reverse scan count
+} SScanPhysiNode;
+
+typedef SScanPhysiNode SSystemTableScanPhysiNode;
+typedef SScanPhysiNode STagScanPhysiNode;
+
+typedef struct STableScanPhysiNode {
+  SScanPhysiNode scan;
+  uint8_t scanFlag;         // denotes reversed scan of data or not
+  STimeWindow scanRange;
+} STableScanPhysiNode;
+
+typedef STableScanPhysiNode STableSeqScanPhysiNode;
+
+typedef struct SProjectPhysiNode {
+  SPhysiNode node;
+  SNodeList* pProjections;
+} SProjectPhysiNode;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_PLANN_NODES_H_*/

include/libs/nodes/querynodes.h

@@ -37,7 +37,7 @@ typedef struct SDataType {
 } SDataType;
 
 typedef struct SExprNode {
-  ENodeType nodeType;
+  ENodeType type;
   SDataType resType;
   char aliasName[TSDB_COL_NAME_LEN];
   SNodeList* pAssociationList;
@@ -50,6 +50,7 @@ typedef enum EColumnType {
 
 typedef struct SColumnNode {
   SExprNode node; // QUERY_NODE_COLUMN
+  uint64_t tableId;
   int16_t colId;
   EColumnType colType; // column or tag
   char dbName[TSDB_DB_NAME_LEN];
@@ -59,6 +60,21 @@ typedef struct SColumnNode {
   SNode* pProjectRef;
 } SColumnNode;
 
+typedef struct SColumnRefNode {
+  ENodeType type;
+  SDataType dataType;
+  int16_t tupleId;
+  int16_t slotId;
+  int16_t columnId;
+} SColumnRefNode;
+
+typedef struct STargetNode {
+  ENodeType type;
+  int16_t tupleId;
+  int16_t slotId;
+  SNode* pExpr;
+} STargetNode;
+
 typedef struct SValueNode {
   SExprNode node; // QUERY_NODE_VALUE
   char* literal;
@@ -72,33 +88,6 @@ typedef struct SValueNode {
   } datum;
 } SValueNode;
 
-typedef enum EOperatorType {
-  // arithmetic operator
-  OP_TYPE_ADD = 1,
-  OP_TYPE_SUB,
-  OP_TYPE_MULTI,
-  OP_TYPE_DIV,
-  OP_TYPE_MOD,
-
-  // comparison operator
-  OP_TYPE_GREATER_THAN,
-  OP_TYPE_GREATER_EQUAL,
-  OP_TYPE_LOWER_THAN,
-  OP_TYPE_LOWER_EQUAL,
-  OP_TYPE_EQUAL,
-  OP_TYPE_NOT_EQUAL,
-  OP_TYPE_IN,
-  OP_TYPE_NOT_IN,
-  OP_TYPE_LIKE,
-  OP_TYPE_NOT_LIKE,
-  OP_TYPE_MATCH,
-  OP_TYPE_NMATCH,
-
-  // json operator
-  OP_TYPE_JSON_GET_VALUE,
-  OP_TYPE_JSON_CONTAINS
-} EOperatorType;
-
 typedef struct SOperatorNode {
   SExprNode node; // QUERY_NODE_OPERATOR
   EOperatorType opType;
@@ -106,26 +95,16 @@ typedef struct SOperatorNode {
   SNode* pRight;
 } SOperatorNode;
 
-typedef enum ELogicConditionType {
-  LOGIC_COND_TYPE_AND,
-  LOGIC_COND_TYPE_OR,
-  LOGIC_COND_TYPE_NOT,
-} ELogicConditionType;
 
 typedef struct SLogicConditionNode {
-  ENodeType type; // QUERY_NODE_LOGIC_CONDITION
+  SExprNode node; // QUERY_NODE_LOGIC_CONDITION
   ELogicConditionType condType;
   SNodeList* pParameterList;
 } SLogicConditionNode;
 
-typedef struct SIsNullCondNode {
-  ENodeType type; // QUERY_NODE_IS_NULL_CONDITION
-  SNode* pExpr;
-  bool isNull;
-} SIsNullCondNode;
-
 typedef struct SNodeListNode {
   ENodeType type; // QUERY_NODE_NODE_LIST
+  SDataType dataType;
   SNodeList* pNodeList;
 } SNodeListNode;
 
@@ -138,7 +117,7 @@ typedef struct SFunctionNode {
 } SFunctionNode;
 
 typedef struct STableNode {
-  ENodeType type;
+  SExprNode node;
   char dbName[TSDB_DB_NAME_LEN];
   char tableName[TSDB_TABLE_NAME_LEN];
   char tableAlias[TSDB_TABLE_NAME_LEN];
@@ -264,6 +243,25 @@ typedef struct SSetOperator {
   SNode* pLimit;
 } SSetOperator;
 
+typedef enum ESqlClause {
+  SQL_CLAUSE_FROM = 1,
+  SQL_CLAUSE_WHERE,
+  SQL_CLAUSE_PARTITION_BY,
+  SQL_CLAUSE_WINDOW,
+  SQL_CLAUSE_GROUP_BY,
+  SQL_CLAUSE_HAVING,
+  SQL_CLAUSE_SELECT,
+  SQL_CLAUSE_ORDER_BY
+} ESqlClause;
+
+void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);
+void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext);
+
+int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, SNodeList** pCols);
+
+typedef bool (*FFuncClassifier)(int32_t funcId);
+int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNodeList** pFuncs);
+
 bool nodesIsExprNode(const SNode* pNode);
 
 bool nodesIsArithmeticOp(const SOperatorNode* pOp);
@@ -273,8 +271,10 @@ bool nodesIsJsonOp(const SOperatorNode* pOp);
 bool nodesIsTimeorderQuery(const SNode* pQuery);
 bool nodesIsTimelineQuery(const SNode* pQuery);
 
+void* nodesGetValueFromNode(SValueNode *pNode);
+
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_QUERY_NODES_H_*/
+#endif /*_TD_QUERY_NODES_H_*/
\ No newline at end of file

source/libs/tdb/src/inc/tdb_db.h → include/libs/parser/newParser.h

@@ -13,33 +13,31 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _TD_TDB_DB_H_
-#define _TD_TDB_DB_H_
-
-#include "tdb_mpool.h"
+#ifndef _TD_NEW_PARSER_H_
+#define _TD_NEW_PARSER_H_
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef struct TDB TDB;
+#include "parser.h"
+
+typedef enum EStmtType {
+  STMT_TYPE_CMD = 1,
+  STMT_TYPE_QUERY
+} EStmtType;
 
-struct TDB {
-  char *      fname;
-  char *      dbname;
-  TDB_MPFILE *mpf;
-  // union {
-  //   TDB_BTREE *btree;
-  //   TDB_HASH * hash;
-  //   TDB_HEAP * heap;
-  // } dbam;  // db access method
-};
+typedef struct SQuery {
+  EStmtType stmtType;
+  SNode* pRoot;
+  int32_t numOfResCols;
+  SSchema* pResSchema;
+} SQuery;
 
-int tdbOpen(TDB **dbpp, const char *fname, const char *dbname, uint32_t flags);
-int tdbClose(TDB *dbp, uint32_t flags);
+int32_t parser(SParseContext* pParseCxt, SQuery* pQuery);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_TDB_DB_H_*/
+#endif /*_TD_NEW_PARSER_H_*/
\ No newline at end of file

include/libs/parser/parsenodes.h

@@ -135,7 +135,7 @@ typedef struct SVgDataBlocks {
   SVgroupInfo vg;
   int32_t     numOfTables;  // number of tables in current submit block
   uint32_t    size;
-  char       *pData;        // SMsgDesc + SSubmitMsg + SSubmitBlk + ...
+  char       *pData;        // SMsgDesc + SSubmitReq + SSubmitBlk + ...
 } SVgDataBlocks;
 
 typedef struct SVnodeModifOpStmtInfo {

include/libs/qcom/query.h

@@ -113,6 +113,7 @@ typedef struct STableMetaOutput {
 typedef struct SDataBuf {
   void     *pData;
   uint32_t  len;
+  void     *handle;
 } SDataBuf;
 
 typedef int32_t (*__async_send_cb_fn_t)(void* param, const SDataBuf* pMsg, int32_t code);

include/util/tnote.h → include/libs/scalar/filter.h

@@ -12,53 +12,41 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-
-#ifndef _TD_UTIL_NOTE_H
-#define _TD_UTIL_NOTE_H
+#ifndef TDENGINE_FILTER_H
+#define TDENGINE_FILTER_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#define MAX_NOTE_LINE_SIZE 66000
-#define NOTE_FILE_NAME_LEN 300
-
-typedef struct {
-  int32_t fileNum;
-  int32_t maxLines;
-  int32_t lines;
-  int32_t flag;
-  int32_t fd;
-  int32_t openInProgress;
-  char    name[NOTE_FILE_NAME_LEN];
-  pthread_mutex_t mutex;
-} SNoteObj;
+typedef struct SFilterInfo SFilterInfo;
+typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **);
 
-extern SNoteObj tsHttpNote;
-extern SNoteObj tsTscNote;
-extern SNoteObj tsInfoNote;
 
-int32_t taosInitNotes();
-void    taosNotePrint(SNoteObj* pNote, const char* const format, ...);
-void    taosNotePrintBuffer(SNoteObj* pNote, char* buffer, int32_t len);
+enum {
+  FLT_OPTION_NO_REWRITE = 1,
+  FLT_OPTION_TIMESTAMP = 2,
+  FLT_OPTION_NEED_UNIQE = 4,
+};
 
-#define nPrintHttp(...)                      \
-  if (tsHttpEnableRecordSql) {               \
-    taosNotePrint(&tsHttpNote, __VA_ARGS__); \
-  }
+typedef struct SFilterColumnParam{
+  int32_t numOfCols;
+  SArray* pDataBlock;
+} SFilterColumnParam;
 
-#define nPrintTsc(...)                      \
-  if (tsTscEnableRecordSql) {               \
-    taosNotePrint(&tsTscNote, __VA_ARGS__); \
-  }
 
-#define nInfo(buffer, len)                         \
-  if (tscEmbeddedInUtil == 1) {                          \
-    taosNotePrintBuffer(&tsInfoNote, buffer, len); \
-  }
+extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t options);
+extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols);
+extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param);
+extern int32_t filterSetDataFromColId(SFilterInfo *info, void *param);
+extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win);
+extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar);
+extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo);
+extern void filterFreeInfo(SFilterInfo *info);
+extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_UTIL_NOTE_H*/
+#endif  // TDENGINE_FILTER_H

include/libs/scalar/scalar.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef TDENGINE_SCALAR_H
+#define TDENGINE_SCALAR_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "function.h"
+#include "nodes.h"
+#include "querynodes.h"
+
+typedef struct SFilterInfo SFilterInfo;
+
+
+int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes);
+int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst);
+int32_t scalarGetOperatorParamNum(EOperatorType type);
+int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type);
+
+int32_t vectorGetConvertType(int32_t type1, int32_t type2);
+int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_SCALAR_H

include/libs/sync/sync.h

@@ -137,22 +137,21 @@ typedef struct {
 
 } SSyncInfo;
 
-// will be defined in syncInt.h, here just for complie
-typedef struct SSyncNode {
-} SSyncNode;
+struct SSyncNode;
+typedef struct SSyncNode SSyncNode;
 
 int32_t syncInit();
 void    syncCleanUp();
 
-int64_t syncStart(const SSyncInfo*);
+int64_t syncStart(const SSyncInfo* pSyncInfo);
 void    syncStop(int64_t rid);
-int32_t syncReconfig(int64_t rid, const SSyncCfg*);
+int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
 
 // int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak);
 int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak);
 
 ESyncState syncGetMyRole(int64_t rid);
-void       syncGetNodesRole(int64_t rid, SNodesRole*);
+void       syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole);
 
 extern int32_t sDebugFlag;
 

include/libs/transport/trpc.h

@@ -64,6 +64,7 @@ typedef struct SRpcInit {
   int8_t   connType;      // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS
   int      idleTime;      // milliseconds, 0 means idle timer is disabled
 
+  bool noPool;  //  create conn pool or not
   // the following is for client app ecurity only
   char *user;     // user name
   char  spi;      // security parameter index
@@ -77,12 +78,21 @@ typedef struct SRpcInit {
   // call back to retrieve the client auth info, for server app only
   int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
 
+  // call back to keep conn or not
+  bool (*pfp)(void *parent, tmsg_t msgType);
+
   void *parent;
 } SRpcInit;
 
-int32_t rpcInit();
+typedef struct {
+  int32_t rpcTimer;
+  int32_t rpcMaxTime;
+  int32_t sver;
+} SRpcCfg;
+
+int32_t rpcInit(SRpcCfg *pCfg);
 void    rpcCleanup();
-void *  rpcOpen(const SRpcInit *pRpc);
+void   *rpcOpen(const SRpcInit *pRpc);
 void    rpcClose(void *);
 void *  rpcMallocCont(int contLen);
 void    rpcFreeCont(void *pCont);

include/os/os.h

@@ -59,6 +59,7 @@ extern "C" {
 #include "osEndian.h"
 #include "osEnv.h"
 #include "osFile.h"
+#include "osLocale.h"
 #include "osLz4.h"
 #include "osMath.h"
 #include "osMemory.h"
@@ -73,6 +74,7 @@ extern "C" {
 #include "osThread.h"
 #include "osTime.h"
 #include "osTimer.h"
+#include "osTimezone.h"
 
 void osInit();
 

include/os/osDir.h

@@ -23,8 +23,8 @@ extern "C" {
 void    taosRemoveDir(const char *dirname);
 int32_t taosDirExist(char *dirname);
 int32_t taosMkDir(const char *dirname);
-void    taosRemoveOldFiles(char *dirname, int32_t keepDays);
-int32_t taosExpandDir(char *dirname, char *outname, int32_t maxlen);
+void    taosRemoveOldFiles(const char *dirname, int32_t keepDays);
+int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen);
 int32_t taosRealPath(char *dirname, int32_t maxlen);
 
 #ifdef __cplusplus

include/os/osEnv.h

@@ -16,16 +16,38 @@
 #ifndef _TD_OS_ENV_H_
 #define _TD_OS_ENV_H_
 
+#include "osSysinfo.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-extern char tsOsName[];
-extern char tsDataDir[];
-extern char tsLogDir[];
-extern char tsScriptDir[];
+typedef struct SOsEnv SOsEnv;
+
 extern char configDir[];
 
+void osInit();
+void osUpdate();
+
+bool   osLogSpaceAvailable();
+int8_t osDaylight();
+
+const char *osLogDir();
+const char *osTempDir();
+const char *osDataDir();
+const char *osName();
+const char *osTimezone();
+const char *osLocale();
+const char *osCharset();
+
+void osSetLogDir(const char *logDir);
+void osSetTempDir(const char *tempDir);
+void osSetDataDir(const char *dataDir);
+void osSetLogReservedSpace(float sizeInGB);
+void osSetTempReservedSpace(float sizeInGB);
+void osSetDataReservedSpace(float sizeInGB);
+void osSetTimezone(const char *timezone);
+
 #ifdef __cplusplus
 }
 #endif

include/os/osLocale.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_OS_LOCALE_H_
+#define _TD_OS_LOCALE_H_
+
+#include "os.h"
+#include "osString.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+char *taosCharsetReplace(char *charsetstr);
+void  taosGetSystemLocale(char *outLocale, char *outCharset);
+void  taosSetSystemLocale(const char *inLocale, const char *inCharSet);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_OS_LOCALE_H_*/

include/os/osSocket.h

@@ -55,7 +55,7 @@ void    taosSetMaskSIGPIPE();
 int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen);
 int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen);
 
-uint32_t    taosInetAddr(char *ipAddr);
+uint32_t    taosInetAddr(const char *ipAddr);
 const char *taosInetNtoa(struct in_addr ipInt);
 
 #if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) 

include/os/osString.h

@@ -45,7 +45,6 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs);
 bool    taosMbsToUcs4(const char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len);
 int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize);
 bool    taosValidateEncodec(const char *encodec);
-char *  taosCharsetReplace(char *charsetstr);
 
 #ifdef __cplusplus
 }

include/os/osSysinfo.h

@@ -21,18 +21,10 @@ extern "C" {
 #endif
 
 #include "os.h"
-  
-#define TSDB_LOCALE_LEN   64
-#define TSDB_TIMEZONE_LEN 96
 
-extern int64_t tsPageSize;
-extern int64_t tsOpenMax;
-extern int64_t tsStreamMax;
-extern int32_t tsNumOfCores;
-extern int32_t tsTotalMemoryMB;
-extern char     tsTimezone[];
-extern char     tsLocale[];
-extern char     tsCharset[];            // default encode string
+#define TD_LOCALE_LEN   64
+#define TD_CHARSET_LEN  64
+#define TD_TIMEZONE_LEN 96
 
 typedef struct {
   int64_t total;
@@ -40,6 +32,19 @@ typedef struct {
   int64_t avail;
 } SDiskSize;
 
+typedef struct SDiskSpace {
+  int64_t   reserved;
+  SDiskSize size;
+} SDiskSpace;
+
+extern int64_t tsPageSize;
+extern int64_t tsOpenMax;
+extern int64_t tsStreamMax;
+extern int32_t tsNumOfCores;
+extern int32_t tsTotalMemoryMB;
+
+
+
 int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
 int32_t taosGetCpuCores();
 void    taosGetSystemInfo();

include/os/osTimezone.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_OS_TIMEZONE_H_
+#define _TD_OS_TIMEZONE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void taosGetSystemTimezone(char *outTimezone);
+void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_OS_TIMEZONE_H_*/

include/util/compare.h

@@ -49,10 +49,19 @@ int32_t WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size,
 
 int32_t taosArrayCompareString(const void *a, const void *b);
 
-int32_t setCompareBytes1(const void *pLeft, const void *pRight);
-int32_t setCompareBytes2(const void *pLeft, const void *pRight);
-int32_t setCompareBytes4(const void *pLeft, const void *pRight);
-int32_t setCompareBytes8(const void *pLeft, const void *pRight);
+int32_t setChkInBytes1(const void *pLeft, const void *pRight);
+int32_t setChkInBytes2(const void *pLeft, const void *pRight);
+int32_t setChkInBytes4(const void *pLeft, const void *pRight);
+int32_t setChkInBytes8(const void *pLeft, const void *pRight);
+
+int32_t setChkNotInBytes1(const void *pLeft, const void *pRight);
+int32_t setChkNotInBytes2(const void *pLeft, const void *pRight);
+int32_t setChkNotInBytes4(const void *pLeft, const void *pRight);
+int32_t setChkNotInBytes8(const void *pLeft, const void *pRight);
+
+int32_t compareChkInString(const void *pLeft, const void *pRight);
+int32_t compareChkNotInString(const void *pLeft, const void *pRight);
+
 
 int32_t compareInt8Val(const void *pLeft, const void *pRight);
 int32_t compareInt16Val(const void *pLeft, const void *pRight);
@@ -74,7 +83,6 @@ int32_t compareStrRegexComp(const void *pLeft, const void *pRight);
 int32_t compareStrRegexCompMatch(const void *pLeft, const void *pRight);
 int32_t compareStrRegexCompNMatch(const void *pLeft, const void *pRight);
 
-int32_t compareFindItemInSet(const void *pLeft, const void *pRight);
 
 int32_t compareInt8ValDesc(const void *pLeft, const void *pRight);
 int32_t compareInt16ValDesc(const void *pLeft, const void *pRight);
@@ -92,6 +100,9 @@ int32_t compareUint64ValDesc(const void *pLeft, const void *pRight);
 int32_t compareLenPrefixedStrDesc(const void *pLeft, const void *pRight);
 int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight);
 
+__compar_fn_t getComparFunc(int32_t type, int32_t optr);
+
+
 #ifdef __cplusplus
 }
 #endif

include/util/taoserror.h

@@ -20,6 +20,8 @@
 extern "C" {
 #endif
 
+// clang-format off
+
 #define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code))))
  
 #define TAOS_SYSTEM_ERROR(code)             (0x80ff0000 | (code))
@@ -71,6 +73,8 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MSG_NOT_PROCESSED             TAOS_DEF_ERROR_CODE(0, 0x0109)
 #define TSDB_CODE_INVALID_PARA                  TAOS_DEF_ERROR_CODE(0, 0x010A)
 #define TSDB_CODE_REPEAT_INIT                   TAOS_DEF_ERROR_CODE(0, 0x010B)
+#define TSDB_CODE_CFG_NOT_FOUND                 TAOS_DEF_ERROR_CODE(0, 0x010C)
+#define TSDB_CODE_INVALID_CFG                   TAOS_DEF_ERROR_CODE(0, 0x010D)
 
 #define TSDB_CODE_REF_NO_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0110)
 #define TSDB_CODE_REF_FULL                      TAOS_DEF_ERROR_CODE(0, 0x0111)
@@ -246,6 +250,8 @@ int32_t* taosGetErrno();
 // mnode-trans
 #define TSDB_CODE_MND_TRANS_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03D0)
 #define TSDB_CODE_MND_TRANS_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x03D1)
+#define TSDB_CODE_MND_TRANS_INVALID_STAGE       TAOS_DEF_ERROR_CODE(0, 0x03D2)
+#define TSDB_CODE_MND_TRANS_CANT_PARALLEL       TAOS_DEF_ERROR_CODE(0, 0x03D4)
 
 // mnode-mq
 #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E0)
@@ -258,6 +264,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_CONSUMER_NOT_EXIST        TAOS_DEF_ERROR_CODE(0, 0x03E7)
 #define TSDB_CODE_MND_UNSUPPORTED_TOPIC         TAOS_DEF_ERROR_CODE(0, 0x03E8)
 #define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E9)
+#define TSDB_CODE_MND_OFFSET_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03EA)
 #define TSDB_CODE_MND_MQ_PLACEHOLDER            TAOS_DEF_ERROR_CODE(0, 0x03F0)
 
 // dnode

include/util/tconfig.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_UTIL_CONFIG_H
-#define _TD_UTIL_CONFIG_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define TSDB_CFG_MAX_NUM    115
-#define TSDB_CFG_PRINT_LEN  23
-#define TSDB_CFG_OPTION_LEN 24
-#define TSDB_CFG_VALUE_LEN  41
-
-#define TSDB_CFG_CTYPE_B_CONFIG    1U   // can be configured from file
-#define TSDB_CFG_CTYPE_B_SHOW      2U   // can displayed by "show configs" commands
-#define TSDB_CFG_CTYPE_B_LOG       4U   // is a log type configuration
-#define TSDB_CFG_CTYPE_B_CLIENT    8U   // can be displayed in the client log
-#define TSDB_CFG_CTYPE_B_OPTION    16U  // can be configured by taos_options function
-#define TSDB_CFG_CTYPE_B_NOT_PRINT 32U  // such as password
-
-#define MAX_FLOAT    100000
-#define MIN_FLOAT    0
-
-enum {
-  TAOS_CFG_CSTATUS_NONE,     // not configured
-  TAOS_CFG_CSTATUS_DEFAULT,  // use system default value
-  TAOS_CFG_CSTATUS_FILE,     // configured from file
-  TAOS_CFG_CSTATUS_OPTION,   // configured by taos_options function
-  TAOS_CFG_CSTATUS_ARG,      // configured by program argument
-};
-
-enum {
-  TAOS_CFG_VTYPE_INT8,
-  TAOS_CFG_VTYPE_INT16,
-  TAOS_CFG_VTYPE_INT32,
-  TAOS_CFG_VTYPE_UINT16,
-  TAOS_CFG_VTYPE_FLOAT,
-  TAOS_CFG_VTYPE_STRING,
-  TAOS_CFG_VTYPE_IPSTR,
-  TAOS_CFG_VTYPE_DIRECTORY,
-  TAOS_CFG_VTYPE_DATA_DIRCTORY,
-  TAOS_CFG_VTYPE_DOUBLE,
-};
-
-enum {
-  TAOS_CFG_UTYPE_NONE,
-  TAOS_CFG_UTYPE_PERCENT,
-  TAOS_CFG_UTYPE_GB,
-  TAOS_CFG_UTYPE_MB,
-  TAOS_CFG_UTYPE_BYTE,
-  TAOS_CFG_UTYPE_SECOND,
-  TAOS_CFG_UTYPE_MS
-};
-
-typedef struct {
-  char *   option;
-  void *   ptr;
-  float    minValue;
-  float    maxValue;
-  int8_t   cfgType;
-  int8_t   cfgStatus;
-  int8_t   unitType;
-  int8_t   valType;
-  int32_t  ptrLength;
-} SGlobalCfg;
-
-extern SGlobalCfg tsGlobalConfig[];
-extern int32_t    tsGlobalConfigNum;
-extern char *     tsCfgStatusStr[];
-
-void    taosReadGlobalLogCfg();
-int32_t taosReadCfgFromFile();
-void    taosPrintCfg();
-void    taosDumpGlobalCfg();
-
-void        taosAddConfigOption(SGlobalCfg cfg);
-SGlobalCfg *taosGetConfigOption(const char *option);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  /*_TD_UTIL_CONFIG_H*/

include/util/tdef.h

@@ -13,6 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+// clang-format off
+
 #ifndef _TD_UTIL_DEF_H
 #define _TD_UTIL_DEF_H
 
@@ -108,33 +110,52 @@ do { \
   (src) = (void *)((char *)src + sizeof(type));\
 } while(0)
 
+typedef enum EOperatorType {
+  // arithmetic operator
+  OP_TYPE_ADD = 1,
+  OP_TYPE_SUB,
+  OP_TYPE_MULTI,
+  OP_TYPE_DIV,
+  OP_TYPE_MOD,
+
+  // bit operator
+  OP_TYPE_BIT_AND,
+  OP_TYPE_BIT_OR,
+
+  // comparison operator
+  OP_TYPE_GREATER_THAN,
+  OP_TYPE_GREATER_EQUAL,
+  OP_TYPE_LOWER_THAN,
+  OP_TYPE_LOWER_EQUAL,
+  OP_TYPE_EQUAL,
+  OP_TYPE_NOT_EQUAL,
+  OP_TYPE_IN,
+  OP_TYPE_NOT_IN,
+  OP_TYPE_LIKE,
+  OP_TYPE_NOT_LIKE,
+  OP_TYPE_MATCH,
+  OP_TYPE_NMATCH,
+  OP_TYPE_IS_NULL,
+  OP_TYPE_IS_NOT_NULL,
+  OP_TYPE_IS_TRUE,
+  OP_TYPE_IS_FALSE,
+  OP_TYPE_IS_UNKNOWN,
+  OP_TYPE_IS_NOT_TRUE,
+  OP_TYPE_IS_NOT_FALSE,
+  OP_TYPE_IS_NOT_UNKNOWN,
+
+  // json operator
+  OP_TYPE_JSON_GET_VALUE,
+  OP_TYPE_JSON_CONTAINS
+} EOperatorType;
+
+
+typedef enum ELogicConditionType {
+  LOGIC_COND_TYPE_AND,
+  LOGIC_COND_TYPE_OR,
+  LOGIC_COND_TYPE_NOT,
+} ELogicConditionType;
 
-// TODO: check if below is necessary
-#define TSDB_RELATION_INVALID     0
-#define TSDB_RELATION_LESS        1
-#define TSDB_RELATION_GREATER     2
-#define TSDB_RELATION_EQUAL       3
-#define TSDB_RELATION_LESS_EQUAL  4
-#define TSDB_RELATION_GREATER_EQUAL 5
-#define TSDB_RELATION_NOT_EQUAL   6
-#define TSDB_RELATION_LIKE        7
-#define TSDB_RELATION_ISNULL      8
-#define TSDB_RELATION_NOTNULL     9
-#define TSDB_RELATION_IN          10
-
-#define TSDB_RELATION_AND         11
-#define TSDB_RELATION_OR          12
-#define TSDB_RELATION_NOT         13
-
-#define TSDB_RELATION_MATCH       14
-#define TSDB_RELATION_NMATCH      15
-
-#define TSDB_BINARY_OP_ADD        4000
-#define TSDB_BINARY_OP_SUBTRACT   4001
-#define TSDB_BINARY_OP_MULTIPLY   4002
-#define TSDB_BINARY_OP_DIVIDE     4003
-#define TSDB_BINARY_OP_REMAINDER  4004
-#define TSDB_BINARY_OP_CONCAT     4005
 
 #define FUNCTION_CEIL        4500
 #define FUNCTION_FLOOR       4501
@@ -146,9 +167,6 @@ do { \
 #define FUNCTION_LTRIM       4802
 #define FUNCTION_RTRIM       4803
 
-#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN))
-#define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER))
-
 #define TSDB_NAME_DELIMITER_LEN   1
 
 #define TSDB_UNI_LEN              24
@@ -180,6 +198,7 @@ do { \
 #define TSDB_TOPIC_FNAME_LEN      TSDB_TABLE_FNAME_LEN
 #define TSDB_CONSUMER_GROUP_LEN   192
 #define TSDB_SUBSCRIBE_KEY_LEN    (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2)
+#define TSDB_PARTITION_KEY_LEN    (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2)
 #define TSDB_COL_NAME_LEN         65
 #define TSDB_MAX_SAVED_SQL_LEN    TSDB_MAX_COLUMNS * 64
 #define TSDB_MAX_SQL_LEN          TSDB_PAYLOAD_SIZE
@@ -214,6 +233,10 @@ do { \
 #define TSDB_SHOW_SUBQUERY_LEN    1000
 #define TSDB_SLOW_QUERY_SQL_LEN   512
 
+#define TSDB_TRANS_STAGE_LEN      12
+#define TSDB_TRANS_TYPE_LEN       16
+#define TSDB_TRANS_ERROR_LEN      64
+
 #define TSDB_STEP_NAME_LEN        32
 #define TSDB_STEP_DESC_LEN        128
 
@@ -329,7 +352,6 @@ do { \
 #define TSDB_QUERY_TYPE_NON_TYPE        0x00u     // none type
 #define TSDB_QUERY_TYPE_FREE_RESOURCE   0x01u     // free qhandle at vnode
 
-
 #define TSDB_META_COMPACT_RATIO         0       // disable tsdb meta compact by default
 
 

include/util/thash.h

@@ -55,6 +55,8 @@ uint32_t taosIntHash_64(const char *key, uint32_t len);
 
 _hash_fn_t taosGetDefaultHashFunction(int32_t type);
 
+_equal_fn_t taosGetDefaultEqualFunction(int32_t type);
+
 typedef struct SHashNode {
   struct SHashNode *next;
   uint32_t          hashVal;     // the hash value of key
@@ -258,6 +260,8 @@ void* taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen);
  */
 void taosHashRelease(SHashObj *pHashObj, void *p);
 
+void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp);
+
 
 #ifdef __cplusplus
 }

include/util/tjson.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_UTIL_JSON_H_
+#define _TD_UTIL_JSON_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "os.h"
+
+typedef void SJson;
+
+SJson* tjsonCreateObject();
+void tjsonDelete(SJson* pJson);
+
+SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName);
+
+int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t number);
+int32_t tjsonAddDoubleToObject(SJson* pJson, const char* pName, const double number);
+int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal);
+int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem);
+int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem);
+
+typedef int32_t (*FToJson)(const void* pObj, SJson* pJson);
+
+int32_t tjsonAddObject(SJson* pJson, const char* pName, FToJson func, const void* pObj);
+int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj);
+
+typedef int32_t (*FFromJson)(const SJson* pJson, void* pObj);
+
+char* tjsonToString(const SJson* pJson);
+char* tjsonToUnformattedString(const SJson* pJson);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_JSON_H_*/

include/util/tlog.h

@@ -22,8 +22,8 @@
 extern "C" {
 #endif
 
-// log
-extern int8_t  tsAsyncLog;
+extern bool    tsLogInited;
+extern bool    tsAsyncLog;
 extern int32_t tsNumOfLogLines;
 extern int32_t tsLogKeepDays;
 extern int32_t dDebugFlag;
@@ -32,9 +32,6 @@ extern int32_t mDebugFlag;
 extern int32_t cDebugFlag;
 extern int32_t jniDebugFlag;
 extern int32_t tmrDebugFlag;
-extern int32_t httpDebugFlag;
-extern int32_t mqttDebugFlag;
-extern int32_t monDebugFlag;
 extern int32_t uDebugFlag;
 extern int32_t rpcDebugFlag;
 extern int32_t qDebugFlag;
@@ -42,23 +39,23 @@ extern int32_t wDebugFlag;
 extern int32_t sDebugFlag;
 extern int32_t tsdbDebugFlag;
 extern int32_t tqDebugFlag;
-extern int32_t cqDebugFlag;
-extern int32_t debugFlag;
-
-#define DEBUG_FATAL 1U
-#define DEBUG_ERROR DEBUG_FATAL
-#define DEBUG_WARN 2U
-#define DEBUG_INFO DEBUG_WARN
-#define DEBUG_DEBUG 4U
-#define DEBUG_TRACE 8U
-#define DEBUG_DUMP 16U
+extern int32_t fsDebugFlag;
 
+#define DEBUG_FATAL  1U
+#define DEBUG_ERROR  DEBUG_FATAL
+#define DEBUG_WARN   2U
+#define DEBUG_INFO   DEBUG_WARN
+#define DEBUG_DEBUG  4U
+#define DEBUG_TRACE  8U
+#define DEBUG_DUMP   16U
 #define DEBUG_SCREEN 64U
-#define DEBUG_FILE 128U
+#define DEBUG_FILE   128U
 
-int32_t taosInitLog(char *logName, int32_t numOfLogLines, int32_t maxFiles);
+int32_t taosInitLog(const char *logName, int32_t maxFiles);
 void    taosCloseLog();
 void    taosResetLog();
+void    taosSetAllDebugFlag(int32_t flag);
+void    taosDumpData(unsigned char *msg, int32_t len);
 
 void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...)
 #ifdef __GNUC__
@@ -72,8 +69,6 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, .
 #endif
     ;
 
-void taosDumpData(unsigned char *msg, int32_t len);
-
 #ifdef __cplusplus
 }
 #endif

include/util/ttimer.h

@@ -26,7 +26,6 @@ typedef void *tmr_h;
 typedef void (*TAOS_TMR_CALLBACK)(void *, void *);
 
 extern int      taosTmrThreads;
-extern uint32_t tsMaxTmrCtrl;
 
 #define MSECONDS_PER_TICK 5
 

include/util/types.h

@@ -66,6 +66,7 @@ static FORCE_INLINE double taos_align_get_double(const char *pBuf) {
 // #else
   #define GET_FLOAT_VAL(x)       (*(float *)(x))
   #define GET_DOUBLE_VAL(x)      (*(double *)(x))
+  #define SET_BIGINT_VAL(x, y)  { (*(int64_t *)(x))  = (int64_t)(y);       }
   #define SET_FLOAT_VAL(x, y)  { (*(float *)(x))  = (float)(y);       }
   #define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y);      }
   #define SET_FLOAT_PTR(x, y)  { (*(float *)(x))  = (*(float *)(y));  }

include/util/version.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_UTIL_VERSION_H
+#define _TD_UTIL_VERSION_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern char version[];
+extern char compatible_version[];
+extern char gitinfo[];
+extern char gitinfoOfInternal[];
+extern char buildinfo[];
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_VERSION_H*/

source/client/CMakeLists.txt

@@ -8,7 +8,7 @@ target_include_directories(
 target_link_libraries(
     taos
     INTERFACE api
-    PRIVATE os util common transport parser planner catalog scheduler function qcom
+    PRIVATE os util common transport parser planner catalog scheduler function qcom config
 )
 
 if(${BUILD_TEST})

source/client/inc/clientInt.h

@@ -20,17 +20,19 @@
 extern "C" {
 #endif
 
-#include "taos.h"
 #include "common.h"
-#include "tmsg.h"
+#include "parser.h"
+#include "query.h"
+#include "taos.h"
 #include "tdef.h"
 #include "tep.h"
 #include "thash.h"
 #include "tlist.h"
+#include "tmsg.h"
 #include "tmsgtype.h"
 #include "trpc.h"
-#include "query.h"
-#include "parser.h"
+
+#include "config.h"
 
 #define CHECK_CODE_GOTO(expr, label) \
   do {                               \
@@ -46,12 +48,12 @@ extern "C" {
 typedef struct SAppInstInfo SAppInstInfo;
 
 typedef struct SHbConnInfo {
-  void         *param;
-  SClientHbReq *req;
+  void*         param;
+  SClientHbReq* req;
 } SHbConnInfo;
 
 typedef struct SAppHbMgr {
-  char   *key;
+  char* key;
   // statistics
   int32_t reportCnt;
   int32_t connKeyCnt;
@@ -62,15 +64,13 @@ typedef struct SAppHbMgr {
   // connection
   SAppInstInfo* pAppInstInfo;
   // info
-  SHashObj* activeInfo;    // hash<SClientHbKey, SClientHbReq>
-  SHashObj* connInfo;      // hash<SClientHbKey, SHbConnInfo>
+  SHashObj* activeInfo;  // hash<SClientHbKey, SClientHbReq>
+  SHashObj* connInfo;    // hash<SClientHbKey, SHbConnInfo>
 } SAppHbMgr;
 
+typedef int32_t (*FHbRspHandle)(struct SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp);
 
-typedef int32_t (*FHbRspHandle)(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp);
-
-typedef int32_t (*FHbReqHandle)(SClientHbKey *connKey, void* param, SClientHbReq *req);
-
+typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req);
 
 typedef struct SClientHbMgr {
   int8_t inited;
@@ -83,63 +83,62 @@ typedef struct SClientHbMgr {
   FHbRspHandle    rspHandle[HEARTBEAT_TYPE_MAX];
 } SClientHbMgr;
 
-
 typedef struct SQueryExecMetric {
-  int64_t      start;    // start timestamp
-  int64_t      parsed;   // start to parse
-  int64_t      send;     // start to send to server
-  int64_t      rsp;      // receive response from server
+  int64_t start;   // start timestamp
+  int64_t parsed;  // start to parse
+  int64_t send;    // start to send to server
+  int64_t rsp;     // receive response from server
 } SQueryExecMetric;
 
 typedef struct SInstanceSummary {
-  uint64_t     numOfInsertsReq;
-  uint64_t     numOfInsertRows;
-  uint64_t     insertElapsedTime;
-  uint64_t     insertBytes;         // submit to tsdb since launched.
-
-  uint64_t     fetchBytes;
-  uint64_t     queryElapsedTime;
-  uint64_t     numOfSlowQueries;
-  uint64_t     totalRequests;
-  uint64_t     currentRequests;      // the number of SRequestObj
+  uint64_t numOfInsertsReq;
+  uint64_t numOfInsertRows;
+  uint64_t insertElapsedTime;
+  uint64_t insertBytes;  // submit to tsdb since launched.
+
+  uint64_t fetchBytes;
+  uint64_t queryElapsedTime;
+  uint64_t numOfSlowQueries;
+  uint64_t totalRequests;
+  uint64_t currentRequests;  // the number of SRequestObj
 } SInstanceSummary;
 
 typedef struct SHeartBeatInfo {
-  void  *pTimer;   // timer, used to send request msg to mnode
+  void* pTimer;  // timer, used to send request msg to mnode
 } SHeartBeatInfo;
 
 struct SAppInstInfo {
-  int64_t          numOfConns;
-  SCorEpSet        mgmtEp;
-  SInstanceSummary summary;
-  SList            *pConnList;  // STscObj linked list
-  int64_t          clusterId;
-  void             *pTransporter;
-  struct SAppHbMgr *pAppHbMgr;
+  int64_t           numOfConns;
+  SCorEpSet         mgmtEp;
+  SInstanceSummary  summary;
+  SList*            pConnList;  // STscObj linked list
+  int64_t           clusterId;
+  void*             pTransporter;
+  struct SAppHbMgr* pAppHbMgr;
 };
 
 typedef struct SAppInfo {
   int64_t         startTime;
   char            appName[TSDB_APP_NAME_LEN];
-  char           *ep;
+  char*           ep;
   int32_t         pid;
   int32_t         numOfThreads;
-  SHashObj       *pInstMap;
+  SHashObj*       pInstMap;
   pthread_mutex_t mutex;
 } SAppInfo;
 
 typedef struct STscObj {
-  char             user[TSDB_USER_LEN];
-  char             pass[TSDB_PASSWORD_LEN];
-  char             db[TSDB_DB_FNAME_LEN];
-  char             ver[128];
-  int32_t          acctId;
-  uint32_t         connId;
-  int32_t          connType;
-  uint64_t         id;       // ref ID returned by taosAddRef
-  pthread_mutex_t  mutex;     // used to protect the operation on db
-  int32_t          numOfReqs; // number of sqlObj bound to this connection
-  SAppInstInfo    *pAppInfo;
+  char            user[TSDB_USER_LEN];
+  char            pass[TSDB_PASSWORD_LEN];
+  char            db[TSDB_DB_FNAME_LEN];
+  char            ver[128];
+  int32_t         acctId;
+  uint32_t        connId;
+  int32_t         connType;
+  uint64_t        id;         // ref ID returned by taosAddRef
+  pthread_mutex_t mutex;      // used to protect the operation on db
+  int32_t         numOfReqs;  // number of sqlObj bound to this connection
+  SAppInstInfo*   pAppInfo;
 } STscObj;
 
 typedef struct SMqConsumer {
@@ -147,49 +146,49 @@ typedef struct SMqConsumer {
 } SMqConsumer;
 
 typedef struct SReqResultInfo {
-  const char  *pRspMsg;
-  const char  *pData;
-  TAOS_FIELD  *fields;
-  uint32_t     numOfCols;
-  int32_t     *length;
-  TAOS_ROW     row;
-  char       **pCol;
-  uint32_t     numOfRows;
-  uint64_t     totalRows;
-  uint32_t     current;
-  bool         completed;
+  const char* pRspMsg;
+  const char* pData;
+  TAOS_FIELD* fields;
+  uint32_t    numOfCols;
+  int32_t*    length;
+  TAOS_ROW    row;
+  char**      pCol;
+  uint32_t    numOfRows;
+  uint64_t    totalRows;
+  uint32_t    current;
+  bool        completed;
 } SReqResultInfo;
 
 typedef struct SShowReqInfo {
-  int64_t         execId;        // showId/queryId
-  int32_t         vgId;
-  SArray         *pArray;        // SArray<SVgroupInfo>
-  int32_t         currentIndex;  // current accessed vgroup index.
+  int64_t execId;  // showId/queryId
+  int32_t vgId;
+  SArray* pArray;        // SArray<SVgroupInfo>
+  int32_t currentIndex;  // current accessed vgroup index.
 } SShowReqInfo;
 
 typedef struct SRequestSendRecvBody {
-  tsem_t            rspSem;        // not used now
+  tsem_t            rspSem;  // not used now
   void*             fp;
-  SShowReqInfo      showInfo;      // todo this attribute will be removed after the query framework being completed.
+  SShowReqInfo      showInfo;  // todo this attribute will be removed after the query framework being completed.
   SDataBuf          requestMsg;
-  struct SSchJob   *pQueryJob;     // query job, created according to sql query DAG.
-  struct SQueryDag *pDag;          // the query dag, generated according to the sql statement.
+  struct SSchJob*   pQueryJob;  // query job, created according to sql query DAG.
+  struct SQueryDag* pDag;       // the query dag, generated according to the sql statement.
   SReqResultInfo    resInfo;
 } SRequestSendRecvBody;
 
-#define ERROR_MSG_BUF_DEFAULT_SIZE  512
+#define ERROR_MSG_BUF_DEFAULT_SIZE 512
 
 typedef struct SRequestObj {
-  uint64_t         requestId;
-  int32_t          type;   // request type
-  STscObj         *pTscObj;
-  char            *sqlstr;  // sql string
-  int32_t          sqlLen;
-  int64_t          self;
-  char            *msgBuf;
-  void            *pInfo;   // sql parse info, generated by parser module
-  int32_t          code;
-  SQueryExecMetric metric;
+  uint64_t             requestId;
+  int32_t              type;  // request type
+  STscObj*             pTscObj;
+  char*                sqlstr;  // sql string
+  int32_t              sqlLen;
+  int64_t              self;
+  char*                msgBuf;
+  void*                pInfo;  // sql parse info, generated by parser module
+  int32_t              code;
+  SQueryExecMetric     metric;
   SRequestSendRecvBody body;
 } SRequestObj;
 
@@ -198,51 +197,52 @@ extern int32_t  clientReqRefPool;
 extern int32_t  clientConnRefPool;
 
 extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code);
-int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code);
+int           genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code);
 SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj);
 
-int   taos_init();
+int taos_init();
 
-void* createTscObj(const char* user, const char* auth, const char *db, SAppInstInfo* pAppInfo);
-void  destroyTscObj(void*pObj);
+void* createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo);
+void  destroyTscObj(void* pObj);
 
 uint64_t generateRequestId();
 
-void *createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type);
+void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type);
 void  destroyRequest(SRequestObj* pRequest);
 
-char *getDbOfConnection(STscObj* pObj);
+char* getDbOfConnection(STscObj* pObj);
 void  setConnectionDB(STscObj* pTscObj, const char* db);
 
 void taos_init_imp(void);
-int  taos_options_imp(TSDB_OPTION option, const char *str);
+int  taos_options_imp(TSDB_OPTION option, const char* str);
 
-void* openTransporter(const char *user, const char *auth, int32_t numOfThreads);
+void* openTransporter(const char* user, const char* auth, int32_t numOfThreads);
 
+bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType);
 void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet);
 
 void initMsgHandleFp();
 
-TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port);
+TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db,
+                            uint16_t port);
 
-void *doFetchRow(SRequestObj* pRequest);
+void* doFetchRow(SRequestObj* pRequest);
 
-void  setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows);
+void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows);
 
-
-int32_t buildRequest(STscObj *pTscObj, const char *sql, int sqlLen, SRequestObj** pRequest);
+int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest);
 
 int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery);
 
-// --- heartbeat 
+// --- heartbeat
 // global, called by mgmt
 int  hbMgrInit();
 void hbMgrCleanUp();
 int  hbHandleRsp(SClientHbBatchRsp* hbRsp);
 
 // cluster level
-SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key);
-void appHbMgrCleanup(void);
+SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key);
+void       appHbMgrCleanup(void);
 
 // conn level
 int  hbRegisterConn(SAppHbMgr* pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType);
@@ -254,6 +254,12 @@ int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* v
 void hbMgrInitMqHbRspHandle();
 
 
+// config
+int32_t tscInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl);
+int32_t tscInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl);
+
+extern SConfig *tscCfg;
+
 #ifdef __cplusplus
 }
 #endif

source/client/src/clientCfg.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "clientInt.h"
+#include "ulog.h"
+
+// todo refact
+SConfig *tscCfg;
+
+static int32_t tscLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl) {
+  char cfgDir[PATH_MAX] = {0};
+  char cfgFile[PATH_MAX + 100] = {0};
+
+  taosExpandDir(inputCfgDir, cfgDir, PATH_MAX);
+  snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir);
+
+  if (cfgLoad(pConfig, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) {
+    uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr());
+    return -1;
+  }
+
+  if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, cfgFile) != 0) {
+    if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, cfgDir) != 0) {
+      uError("failed to load from config file:%s since %s\n", cfgFile, terrstr());
+      return -1;
+    }
+  }
+
+  if (cfgLoad(pConfig, CFG_STYPE_ENV_FILE, envFile) != 0) {
+    uError("failed to load from env file:%s since %s\n", envFile, terrstr());
+    return -1;
+  }
+
+  if (cfgLoad(pConfig, CFG_STYPE_ENV_VAR, NULL) != 0) {
+    uError("failed to load from global env variables since %s\n", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+static int32_t tscAddLogCfg(SConfig *pCfg) {
+  if (cfgAddDir(pCfg, "logDir", "/var/log/taos") != 0) return -1;
+  if (cfgAddBool(pCfg, "asyncLog", 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "numOfLogLines", 10000000, 1000, 2000000000) != 0) return -1;
+  if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000) != 0) return -1;
+  if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "cDebugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "jniDebugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "tmrDebugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "uDebugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "rpcDebugFlag", 0, 0, 255) != 0) return -1;
+  return 0;
+}
+
+static int32_t tscSetLogCfg(SConfig *pCfg) {
+  osSetLogDir(cfgGetItem(pCfg, "logDir")->str);
+  tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval;
+  tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32;
+  tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32;
+  cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32;
+  jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32;
+  tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32;
+  uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32;
+  rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32;
+
+  int32_t debugFlag = cfgGetItem(pCfg, "debugFlag")->i32;
+  taosSetAllDebugFlag(debugFlag);
+  return 0;
+}
+
+int32_t tscInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl) {
+  if (tsLogInited) return 0;
+
+  SConfig *pCfg = cfgInit();
+  if (pCfg == NULL) return -1;
+
+  if (tscAddLogCfg(pCfg) != 0) {
+    printf("failed to add log cfg since %s\n", terrstr());
+    cfgCleanup(pCfg);
+    return -1;
+  }
+
+  if (tscLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) {
+    printf("failed to load log cfg since %s\n", terrstr());
+    cfgCleanup(pCfg);
+    return -1;
+  }
+
+  if (tscSetLogCfg(pCfg) != 0) {
+    printf("failed to set log cfg since %s\n", terrstr());
+    cfgCleanup(pCfg);
+    return -1;
+  }
+
+  const int32_t maxLogFileNum = 10;
+  if (taosInitLog("taoslog", maxLogFileNum) != 0) {
+    printf("failed to init log file since %s\n", terrstr());
+    cfgCleanup(pCfg);
+    return -1;
+  }
+
+  cfgDumpCfg(pCfg);
+  cfgCleanup(pCfg);
+  return 0;
+}
+
+static int32_t tscAddEpCfg(SConfig *pCfg) {
+  char defaultFqdn[TSDB_FQDN_LEN] = {0};
+  if (taosGetFqdn(defaultFqdn) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+  if (cfgAddString(pCfg, "fqdn", defaultFqdn) != 0) return -1;
+
+  int32_t defaultServerPort = 6030;
+  if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056) != 0) return -1;
+
+  char defaultFirstEp[TSDB_EP_LEN] = {0};
+  char defaultSecondEp[TSDB_EP_LEN] = {0};
+  snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort);
+  snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort);
+  if (cfgAddString(pCfg, "firstEp", defaultFirstEp) != 0) return -1;
+  if (cfgAddString(pCfg, "secondEp", defaultSecondEp) != 0) return -1;
+
+  return 0;
+}
+
+static int32_t tscAddCfg(SConfig *pCfg) {
+  if (tscAddEpCfg(pCfg) != 0) return -1;
+
+  // if (cfgAddString(pCfg, "buildinfo", buildinfo) != 0) return -1;
+  // if (cfgAddString(pCfg, "gitinfo", gitinfo) != 0) return -1;
+  // if (cfgAddString(pCfg, "version", version) != 0) return -1;
+
+  // if (cfgAddDir(pCfg, "dataDir", osDataDir()) != 0) return -1;
+  if (cfgAddTimezone(pCfg, "timezone", "") != 0) return -1;
+  if (cfgAddLocale(pCfg, "locale", "") != 0) return -1;
+  if (cfgAddCharset(pCfg, "charset", "") != 0) return -1;
+  if (cfgAddInt32(pCfg, "numOfCores", 1, 1, 100000) != 0) return -1;
+  if (cfgAddInt32(pCfg, "numOfCommitThreads", 4, 1, 1000) != 0) return -1;
+  // if (cfgAddBool(pCfg, "telemetryReporting", 0) != 0) return -1;
+  if (cfgAddBool(pCfg, "enableCoreFile", 0) != 0) return -1;
+  // if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536) != 0) return -1;
+  if (cfgAddInt32(pCfg, "statusInterval", 1, 1, 30) != 0) return -1;
+  if (cfgAddFloat(pCfg, "numOfThreadsPerCore", 1, 0, 10) != 0) return -1;
+  if (cfgAddFloat(pCfg, "ratioOfQueryCores", 1, 0, 5) != 0) return -1;
+  if (cfgAddInt32(pCfg, "shellActivityTimer", 3, 1, 120) != 0) return -1;
+  if (cfgAddInt32(pCfg, "rpcTimer", 300, 100, 3000) != 0) return -1;
+  if (cfgAddInt32(pCfg, "rpcMaxTime", 600, 100, 7200) != 0) return -1;
+  if (cfgAddInt32(pCfg, "maxConnections", 50000, 1, 100000) != 0) return -1;
+  return 0;
+}
+
+int32_t tscCheckCfg(SConfig *pCfg) {
+  bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval;
+  taosSetCoreDump(enableCore);
+
+  return 0;
+}
+
+SConfig *tscInitCfgImp(const char *cfgDir, const char *envFile, const char *apolloUrl) {
+  SConfig *pCfg = cfgInit();
+  if (pCfg == NULL) return NULL;
+
+  if (tscAddCfg(pCfg) != 0) {
+    uError("failed to init tsc cfg since %s", terrstr());
+    cfgCleanup(pCfg);
+    return NULL;
+  }
+
+  if (tscLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) {
+    printf("failed to load tsc cfg since %s\n", terrstr());
+    cfgCleanup(pCfg);
+    return NULL;
+  }
+
+  if (tscCheckCfg(pCfg) != 0) {
+    uError("failed to check cfg since %s", terrstr());
+    cfgCleanup(pCfg);
+    return NULL;
+  }
+
+  cfgDumpCfg(pCfg);
+  return pCfg;
+}
+
+int32_t tscInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl) {
+  tscCfg = tscInitCfgImp(cfgDir, envFile, apolloUrl);
+  if (tscCfg == NULL) return -1;
+
+  return 0;
+}
\ No newline at end of file

source/client/src/clientEnv.c

@@ -13,33 +13,30 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "os.h"
 #include "catalog.h"
 #include "clientInt.h"
 #include "clientLog.h"
+#include "os.h"
 #include "query.h"
 #include "scheduler.h"
-#include "tmsg.h"
 #include "tcache.h"
-#include "tconfig.h"
 #include "tglobal.h"
-#include "tnote.h"
+#include "tmsg.h"
 #include "tref.h"
 #include "trpc.h"
 #include "ttime.h"
-#include "ttimezone.h"
 
 #define TSC_VAR_NOT_RELEASE 1
-#define TSC_VAR_RELEASED    0
+#define TSC_VAR_RELEASED 0
 
-SAppInfo   appInfo;
-int32_t    clientReqRefPool  = -1;
-int32_t    clientConnRefPool = -1;
+SAppInfo appInfo;
+int32_t  clientReqRefPool = -1;
+int32_t  clientConnRefPool = -1;
 
 static pthread_once_t tscinit = PTHREAD_ONCE_INIT;
-volatile int32_t tscInitRes = 0;
+volatile int32_t      tscInitRes = 0;
 
-static void registerRequest(SRequestObj* pRequest) {
+static void registerRequest(SRequestObj *pRequest) {
   STscObj *pTscObj = (STscObj *)taosAcquireRef(clientConnRefPool, pRequest->pTscObj->id);
   assert(pTscObj != NULL);
 
@@ -53,69 +50,58 @@ static void registerRequest(SRequestObj* pRequest) {
 
     int32_t total = atomic_add_fetch_32(&pSummary->totalRequests, 1);
     int32_t currentInst = atomic_add_fetch_32(&pSummary->currentRequests, 1);
-    tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 ", current:%d, app current:%d, total:%d, reqId:0x%"PRIx64, pRequest->self,
-             pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
+    tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64
+             ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64,
+             pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
   }
 }
 
-static void deregisterRequest(SRequestObj* pRequest) {
+static void deregisterRequest(SRequestObj *pRequest) {
   assert(pRequest != NULL);
 
-  STscObj* pTscObj = pRequest->pTscObj;
-  SInstanceSummary* pActivity = &pTscObj->pAppInfo->summary;
+  STscObj *         pTscObj = pRequest->pTscObj;
+  SInstanceSummary *pActivity = &pTscObj->pAppInfo->summary;
 
   int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1);
   int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1);
 
   int64_t duration = taosGetTimestampMs() - pRequest->metric.start;
-  tscDebug("0x%"PRIx64" free Request from connObj: 0x%"PRIx64", reqId:0x%"PRIx64" elapsed:%"PRIu64" ms, current:%d, app current:%d", pRequest->self, pTscObj->id,
-      pRequest->requestId, duration, num, currentInst);
+  tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64
+           " ms, current:%d, app current:%d",
+           pRequest->self, pTscObj->id, pRequest->requestId, duration, num, currentInst);
   taosReleaseRef(clientConnRefPool, pTscObj->id);
 }
 
-static void tscInitLogFile() {
-  taosReadGlobalLogCfg();
-  if (mkdir(tsLogDir, 0755) != 0 && errno != EEXIST) {
-    printf("failed to create log dir:%s\n", tsLogDir);
-  }
-
-  const char    *defaultLogFileNamePrefix = "taoslog";
-  const int32_t  maxLogFileNum = 10;
 
-  char temp[128] = {0};
-  sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix);
-  if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) {
-    printf("failed to open log file in directory:%s\n", tsLogDir);
-  }
-}
 
 // todo close the transporter properly
-void closeTransporter(STscObj* pTscObj)  {
+void closeTransporter(STscObj *pTscObj) {
   if (pTscObj == NULL || pTscObj->pAppInfo->pTransporter == NULL) {
     return;
   }
 
-  tscDebug("free transporter:%p in connObj: 0x%"PRIx64, pTscObj->pAppInfo->pTransporter, pTscObj->id);
+  tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, pTscObj->id);
   rpcClose(pTscObj->pAppInfo->pTransporter);
 }
 
 // TODO refactor
-void* openTransporter(const char *user, const char *auth, int32_t numOfThread) {
+void *openTransporter(const char *user, const char *auth, int32_t numOfThread) {
   SRpcInit rpcInit;
   memset(&rpcInit, 0, sizeof(rpcInit));
   rpcInit.localPort = 0;
   rpcInit.label = "TSC";
   rpcInit.numOfThreads = numOfThread;
   rpcInit.cfp = processMsgFromServer;
-  rpcInit.sessions = tsMaxConnections;
+  rpcInit.pfp = persistConnForSpecificMsg;
+  rpcInit.sessions = cfgGetItem(tscCfg, "maxConnections")->i32;
   rpcInit.connType = TAOS_CONN_CLIENT;
   rpcInit.user = (char *)user;
-  rpcInit.idleTime = tsShellActivityTimer * 1000;
+  rpcInit.idleTime = cfgGetItem(tscCfg, "shellActivityTimer")->i32 * 1000;
   rpcInit.ckey = "key";
   rpcInit.spi = 1;
   rpcInit.secret = (char *)auth;
 
-  void* pDnodeConn = rpcOpen(&rpcInit);
+  void *pDnodeConn = rpcOpen(&rpcInit);
   if (pDnodeConn == NULL) {
     tscError("failed to init connection to server");
     return NULL;
@@ -130,12 +116,12 @@ void destroyTscObj(void *pObj) {
   SClientHbKey connKey = {.connId = pTscObj->connId, .hbType = pTscObj->connType};
   hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey);
   atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
-  tscDebug("connObj 0x%"PRIx64" destroyed, totalConn:%"PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns);
+  tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns);
   pthread_mutex_destroy(&pTscObj->mutex);
   tfree(pTscObj);
 }
 
-void* createTscObj(const char* user, const char* auth, const char *db, SAppInstInfo* pAppInfo) {
+void *createTscObj(const char *user, const char *auth, const char *db, SAppInstInfo *pAppInfo) {
   STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj));
   if (NULL == pObj) {
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
@@ -153,11 +139,11 @@ void* createTscObj(const char* user, const char* auth, const char *db, SAppInstI
   pthread_mutex_init(&pObj->mutex, NULL);
   pObj->id = taosAddRef(clientConnRefPool, pObj);
 
-  tscDebug("connObj created, 0x%"PRIx64, pObj->id);
+  tscDebug("connObj created, 0x%" PRIx64, pObj->id);
   return pObj;
 }
 
-void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type) {
+void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t type) {
   assert(pObj != NULL);
 
   SRequestObj *pRequest = (SRequestObj *)calloc(1, sizeof(SRequestObj));
@@ -166,20 +152,20 @@ void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t ty
     return NULL;
   }
 
-  pRequest->requestId  = generateRequestId();
+  pRequest->requestId = generateRequestId();
   pRequest->metric.start = taosGetTimestampMs();
 
-  pRequest->type       = type;
-  pRequest->pTscObj    = pObj;
-  pRequest->body.fp    = fp;    // not used it yet
-  pRequest->msgBuf     = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE);
+  pRequest->type = type;
+  pRequest->pTscObj = pObj;
+  pRequest->body.fp = fp;  // not used it yet
+  pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE);
   tsem_init(&pRequest->body.rspSem, 0, 0);
 
   registerRequest(pRequest);
   return pRequest;
 }
 
-static void doFreeReqResultInfo(SReqResultInfo* pResInfo) {
+static void doFreeReqResultInfo(SReqResultInfo *pResInfo) {
   tfree(pResInfo->pRspMsg);
   tfree(pResInfo->length);
   tfree(pResInfo->row);
@@ -187,9 +173,9 @@ static void doFreeReqResultInfo(SReqResultInfo* pResInfo) {
   tfree(pResInfo->fields);
 }
 
-static void doDestroyRequest(void* p) {
+static void doDestroyRequest(void *p) {
   assert(p != NULL);
-  SRequestObj* pRequest = (SRequestObj*)p;
+  SRequestObj *pRequest = (SRequestObj *)p;
 
   assert(RID_VALID(pRequest->self));
 
@@ -208,7 +194,7 @@ static void doDestroyRequest(void* p) {
   tfree(pRequest);
 }
 
-void destroyRequest(SRequestObj* pRequest) {
+void destroyRequest(SRequestObj *pRequest) {
   if (pRequest == NULL) {
     return;
   }
@@ -225,41 +211,46 @@ void taos_init_imp(void) {
   srand(taosGetTimestampSec());
 
   deltaToUtcInitOnce();
-  taosInitGlobalCfg();
-  taosReadCfgFromFile();
 
-  tscInitLogFile();
-  if (taosCheckAndPrintCfg()) {
+  if (tscInitLog(configDir, NULL, NULL) != 0) {
+    tscInitRes = -1;
+    return;
+  }
+
+  if (tscInitCfg(configDir, NULL, NULL) != 0) {
     tscInitRes = -1;
     return;
   }
 
-  taosInitNotes();
   initMsgHandleFp();
   initQueryModuleMsgHandle();
 
-  rpcInit();
+  SRpcCfg rpcCfg = {0};
+  rpcCfg.rpcTimer = cfgGetItem(tscCfg, "rpcTimer")->i32;
+  rpcCfg.rpcMaxTime = cfgGetItem(tscCfg, "rpcMaxTime")->i32;
+  rpcCfg.sver = 30000000;
+  rpcInit(&rpcCfg);
 
   SCatalogCfg cfg = {.maxDBCacheNum = 100, .maxTblCacheNum = 100};
   catalogInit(&cfg);
 
   SSchedulerCfg scfg = {.maxJobNum = 100};
   schedulerInit(&scfg);
-  tscDebug("starting to initialize TAOS driver, local ep: %s", tsLocalEp);
+  tscDebug("starting to initialize TAOS driver");
 
   taosSetCoreDump(true);
 
   initTaskQueue();
 
   clientConnRefPool = taosOpenRef(200, destroyTscObj);
-  clientReqRefPool  = taosOpenRef(40960, doDestroyRequest);
+  clientReqRefPool = taosOpenRef(40960, doDestroyRequest);
 
   taosGetAppName(appInfo.appName, NULL);
   pthread_mutex_init(&appInfo.mutex, NULL);
 
-  appInfo.pid       = taosGetPId();
+  appInfo.pid = taosGetPId();
   appInfo.startTime = taosGetTimestampMs();
-  appInfo.pInstMap  = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+  appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
   tscDebug("client is initialized successfully");
 }
 
@@ -269,6 +260,7 @@ int taos_init() {
 }
 
 int taos_options_imp(TSDB_OPTION option, const char *str) {
+#if 0  
   SGlobalCfg *cfg = NULL;
 
   switch (option) {
@@ -281,7 +273,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
         cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
         tscInfo("set config file directory:%s", str);
       } else {
-        tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
+        tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str,
+                tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
       }
       break;
 
@@ -296,7 +289,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
         cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
         tscInfo("set shellActivityTimer:%d", tsShellActivityTimer);
       } else {
-        tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr);
+        tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str,
+                tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr);
       }
       break;
 
@@ -305,7 +299,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
       assert(cfg != NULL);
 
       size_t len = strlen(str);
-      if (len == 0 || len > TSDB_LOCALE_LEN) {
+      if (len == 0 || len > TD_LOCALE_LEN) {
         tscInfo("Invalid locale:%s, use default", str);
         return -1;
       }
@@ -313,8 +307,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
       if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
         char sep = '.';
 
-        if (strlen(tsLocale) == 0) { // locale does not set yet
-          char* defaultLocale = setlocale(LC_CTYPE, "");
+        if (strlen(tsLocale) == 0) {  // locale does not set yet
+          char *defaultLocale = setlocale(LC_CTYPE, "");
 
           // The locale of the current OS does not be set correctly, so the default locale cannot be acquired.
           // The launch of current system will abort soon.
@@ -323,21 +317,21 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
             return -1;
           }
 
-          tstrncpy(tsLocale, defaultLocale, TSDB_LOCALE_LEN);
+          tstrncpy(tsLocale, defaultLocale, TD_LOCALE_LEN);
         }
 
         // set the user specified locale
         char *locale = setlocale(LC_CTYPE, str);
 
-        if (locale != NULL) { // failed to set the user specified locale
+        if (locale != NULL) {  // failed to set the user specified locale
           tscInfo("locale set, prev locale:%s, new locale:%s", tsLocale, locale);
           cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
-        } else { // set the user specified locale failed, use default LC_CTYPE as current locale
+        } else {  // set the user specified locale failed, use default LC_CTYPE as current locale
           locale = setlocale(LC_CTYPE, tsLocale);
           tscInfo("failed to set locale:%s, current locale:%s", str, tsLocale);
         }
 
-        tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN);
+        tstrncpy(tsLocale, locale, TD_LOCALE_LEN);
 
         char *charset = strrchr(tsLocale, sep);
         if (charset != NULL) {
@@ -352,7 +346,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
               tscInfo("charset changed from %s to %s", tsCharset, charset);
             }
 
-            tstrncpy(tsCharset, charset, TSDB_LOCALE_LEN);
+            tstrncpy(tsCharset, charset, TD_LOCALE_LEN);
             cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
 
           } else {
@@ -360,11 +354,12 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
           }
 
           free(charset);
-        } else { // it may be windows system
+        } else {  // it may be windows system
           tscInfo("charset remains:%s", tsCharset);
         }
       } else {
-        tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
+        tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str,
+                tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
       }
       break;
     }
@@ -375,7 +370,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
       assert(cfg != NULL);
 
       size_t len = strlen(str);
-      if (len == 0 || len > TSDB_LOCALE_LEN) {
+      if (len == 0 || len > TD_LOCALE_LEN) {
         tscInfo("failed to set charset:%s", str);
         return -1;
       }
@@ -388,13 +383,14 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
             tscInfo("charset changed from %s to %s", tsCharset, str);
           }
 
-          tstrncpy(tsCharset, str, TSDB_LOCALE_LEN);
+          tstrncpy(tsCharset, str, TD_LOCALE_LEN);
           cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
         } else {
           tscInfo("charset:%s not valid", str);
         }
       } else {
-        tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
+        tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str,
+                tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
       }
 
       break;
@@ -405,12 +401,13 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
       assert(cfg != NULL);
 
       if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
-        tstrncpy(tsTimezone, str, TSDB_TIMEZONE_LEN);
+        tstrncpy(tsTimezone, str, TD_TIMEZONE_LEN);
         tsSetTimeZone();
         cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
         tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, str);
       } else {
-        tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
+        tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str,
+                tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
       }
       break;
 
@@ -419,7 +416,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
       tscError("Invalid option %d", option);
       return -1;
   }
-
+#endif
   return 0;
 }
 
@@ -434,7 +431,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
  */
 uint64_t generateRequestId() {
   static uint64_t hashId = 0;
-  static int32_t requestSerialId = 0;
+  static int32_t  requestSerialId = 0;
 
   if (hashId == 0) {
     char    uid[64] = {0};
@@ -448,9 +445,9 @@ uint64_t generateRequestId() {
     }
   }
 
-  int64_t ts      = taosGetTimestampMs();
-  uint64_t pid    = taosGetPId();
-  int32_t val     = atomic_add_fetch_32(&requestSerialId, 1);
+  int64_t  ts = taosGetTimestampMs();
+  uint64_t pid = taosGetPId();
+  int32_t  val = atomic_add_fetch_32(&requestSerialId, 1);
 
   uint64_t id = ((hashId & 0x0FFF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF);
   return id;

source/client/src/clientHb.c

@@ -70,62 +70,42 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
     }
   }
 
+  tFreeSUseDbBatchRsp(&batchUseRsp);
   return TSDB_CODE_SUCCESS;
 }
 
 static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) {
-  int32_t msgLen = 0;
   int32_t code = 0;
-  int32_t schemaNum = 0;
-  
-  while (msgLen < valueLen) {
-    STableMetaRsp *rsp = (STableMetaRsp *)((char *)value + msgLen);
 
-    rsp->numOfColumns = ntohl(rsp->numOfColumns);
-    rsp->suid = be64toh(rsp->suid);
-    rsp->dbId = be64toh(rsp->dbId);
-    
+  STableMetaBatchRsp batchMetaRsp = {0};
+  if (tDeserializeSTableMetaBatchRsp(value, valueLen, &batchMetaRsp) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    return -1;
+  }
+
+  int32_t numOfBatchs = taosArrayGetSize(batchMetaRsp.pArray);
+  for (int32_t i = 0; i < numOfBatchs; ++i) {
+    STableMetaRsp *rsp = taosArrayGet(batchMetaRsp.pArray, i);
+
     if (rsp->numOfColumns < 0) {
-      schemaNum = 0;
-      
       tscDebug("hb remove stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName);
-
       catalogRemoveStbMeta(pCatalog, rsp->dbFName, rsp->dbId, rsp->stbName, rsp->suid);
     } else {
       tscDebug("hb update stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName);
-
-      rsp->numOfTags = ntohl(rsp->numOfTags);
-      rsp->sversion = ntohl(rsp->sversion);
-      rsp->tversion = ntohl(rsp->tversion);
-      rsp->tuid = be64toh(rsp->tuid);
-      rsp->vgId = ntohl(rsp->vgId);
-
-      SSchema* pSchema = rsp->pSchema;
-      
-      schemaNum = rsp->numOfColumns + rsp->numOfTags;
-
-      for (int i = 0; i < schemaNum; ++i) {
-        pSchema->bytes = ntohl(pSchema->bytes);
-        pSchema->colId = ntohl(pSchema->colId);
-
-        pSchema++;
-      }
-
-      if (rsp->pSchema[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) {
-        tscError("invalid colId[%d] for the first column in table meta rsp msg", rsp->pSchema[0].colId);
+      if (rsp->pSchemas[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) {
+        tscError("invalid colId[%d] for the first column in table meta rsp msg", rsp->pSchemas[0].colId);
+        tFreeSTableMetaBatchRsp(&batchMetaRsp);
         return TSDB_CODE_TSC_INVALID_VALUE;
-      }      
+      }
 
       catalogUpdateSTableMeta(pCatalog, rsp);
     }
-
-    msgLen += sizeof(STableMetaRsp) + schemaNum * sizeof(SSchema);
   }
 
+  tFreeSTableMetaBatchRsp(&batchMetaRsp);
   return TSDB_CODE_SUCCESS;
 }
 
-
 static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp) {
   SHbConnInfo * info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey));
   if (NULL == info) {
@@ -335,7 +315,7 @@ void hbFreeReq(void *req) {
 
 
 SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) {
-  SClientHbBatchReq* pBatchReq = malloc(sizeof(SClientHbBatchReq));
+  SClientHbBatchReq* pBatchReq = calloc(1, sizeof(SClientHbBatchReq));
   if (pBatchReq == NULL) {
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return NULL;
@@ -415,8 +395,10 @@ static void* hbThreadFunc(void* param) {
         hbClearReqInfo(pAppHbMgr);
         break;
       }
+    
       tSerializeSClientHbBatchReq(buf, tlen, pReq);
-      SMsgSendInfo *pInfo = malloc(sizeof(SMsgSendInfo));
+      SMsgSendInfo *pInfo = calloc(1, sizeof(SMsgSendInfo));
+
       if (pInfo == NULL) {
         terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
         tFreeClientHbBatchReq(pReq, false);

source/client/src/clientMain.c

@@ -7,6 +7,7 @@
 #include "tmsg.h"
 #include "tglobal.h"
 #include "catalog.h"
+#include "version.h"
 
 #define TSC_VAR_NOT_RELEASE 1
 #define TSC_VAR_RELEASED    0
@@ -56,9 +57,7 @@ void taos_cleanup(void) {
 }
 
 TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) {
-  int32_t p = (port != 0) ? port : tsServerPort;
-
-  tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db);
+  tscDebug("try to connect to %s:%u, user:%s db:%s", ip, port, user, db);
   if (user == NULL) {
     user = TSDB_DEFAULT_USER;
   }
@@ -67,7 +66,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha
     pass = TSDB_DEFAULT_PASS;
   }
 
-  return taos_connect_internal(ip, user, pass, NULL, db, p);
+  return taos_connect_internal(ip, user, pass, NULL, db, port);
 }
 
 void taos_close(TAOS* taos) {

source/client/src/clientMsgHandler.c

@@ -20,14 +20,14 @@
 #include "clientLog.h"
 #include "catalog.h"
 
-int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code);
+int32_t (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code);
 
 static void setErrno(SRequestObj* pRequest, int32_t code) {
   pRequest->code = code;
   terrno = code;
 }
 
-int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) {
+int32_t genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) {
   SRequestObj* pRequest = param;
   setErrno(pRequest, code);
 
@@ -36,7 +36,7 @@ int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) {
   return code;
 }
 
-int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
+int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
   SRequestObj* pRequest = param;
   if (code != TSDB_CODE_SUCCESS) {
     free(pMsg->pData);
@@ -45,41 +45,35 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
     return code;
   }
 
-  STscObj *pTscObj = pRequest->pTscObj;
+  STscObj* pTscObj = pRequest->pTscObj;
 
-  SConnectRsp *pConnect = (SConnectRsp *)pMsg->pData;
-  pConnect->acctId    = htonl(pConnect->acctId);
-  pConnect->connId    = htonl(pConnect->connId);
-  pConnect->clusterId = htobe64(pConnect->clusterId);
+  SConnectRsp connectRsp = {0};
+  tDeserializeSConnectRsp(pMsg->pData, pMsg->len, &connectRsp);
+  assert(connectRsp.epSet.numOfEps > 0);
 
-  assert(pConnect->epSet.numOfEps > 0);
-  for(int32_t i = 0; i < pConnect->epSet.numOfEps; ++i) {
-    pConnect->epSet.eps[i].port = htons(pConnect->epSet.eps[i].port);
+  if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &connectRsp.epSet)) {
+    updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &connectRsp.epSet);
   }
 
-  if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &pConnect->epSet)) {
-    updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &pConnect->epSet);
+  for (int32_t i = 0; i < connectRsp.epSet.numOfEps; ++i) {
+    tscDebug("0x%" PRIx64 " epSet.fqdn[%d]:%s port:%d, connObj:0x%" PRIx64, pRequest->requestId, i,
+             connectRsp.epSet.eps[i].fqdn, connectRsp.epSet.eps[i].port, pTscObj->id);
   }
 
-  for (int i = 0; i < pConnect->epSet.numOfEps; ++i) {
-    tscDebug("0x%" PRIx64 " epSet.fqdn[%d]:%s port:%d, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.eps[i].fqdn,
-        pConnect->epSet.eps[i].port, pTscObj->id);
-  }
-
-  pTscObj->connId = pConnect->connId;
-  pTscObj->acctId = pConnect->acctId;
-  tstrncpy(pTscObj->ver, pConnect->sVersion, tListLen(pTscObj->ver));
+  pTscObj->connId = connectRsp.connId;
+  pTscObj->acctId = connectRsp.acctId;
+  tstrncpy(pTscObj->ver, connectRsp.sVersion, tListLen(pTscObj->ver));
 
   // update the appInstInfo
-  pTscObj->pAppInfo->clusterId = pConnect->clusterId;
+  pTscObj->pAppInfo->clusterId = connectRsp.clusterId;
   atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
 
   pTscObj->connType = HEARTBEAT_TYPE_QUERY;
 
-  hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pConnect->connId, pConnect->clusterId, HEARTBEAT_TYPE_QUERY);
+  hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connectRsp.connId, connectRsp.clusterId, HEARTBEAT_TYPE_QUERY);
 
   //  pRequest->body.resInfo.pRspMsg = pMsg->pData;
-  tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, pConnect->clusterId,
+  tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, connectRsp.clusterId,
            pTscObj->pAppInfo->numOfConns);
 
   free(pMsg->pData);
@@ -103,9 +97,9 @@ SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) {
       int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &retrieveReq);
       void*   pReq = malloc(contLen);
       tSerializeSRetrieveTableReq(pReq, contLen, &retrieveReq);
-
       pMsgSendInfo->msgInfo.pData = pReq;
       pMsgSendInfo->msgInfo.len = contLen;
+      pMsgSendInfo->msgInfo.handle = NULL;
     } else {
       SVShowTablesFetchReq* pFetchMsg = calloc(1, sizeof(SVShowTablesFetchReq));
       if (pFetchMsg == NULL) {
@@ -117,6 +111,7 @@ SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) {
 
       pMsgSendInfo->msgInfo.pData = pFetchMsg;
       pMsgSendInfo->msgInfo.len = sizeof(SVShowTablesFetchReq);
+      pMsgSendInfo->msgInfo.handle = NULL;
     }
   } else {
     assert(pRequest != NULL);
@@ -135,39 +130,29 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) {
     return code;
   }
 
-  SShowRsp* pShow = (SShowRsp *)pMsg->pData;
-  pShow->showId   = htobe64(pShow->showId);
-
-  STableMetaRsp *pMetaMsg = &(pShow->tableMeta);
-  pMetaMsg->numOfColumns = htonl(pMetaMsg->numOfColumns);
+  SShowRsp showRsp = {0};
+  tDeserializeSShowRsp(pMsg->pData, pMsg->len, &showRsp);
+  STableMetaRsp *pMetaMsg = &showRsp.tableMeta;
 
-  SSchema* pSchema = pMetaMsg->pSchema;
-  pMetaMsg->tuid = htobe64(pMetaMsg->tuid);
-  for (int i = 0; i < pMetaMsg->numOfColumns; ++i) {
-    pSchema->bytes = htonl(pSchema->bytes);
-    pSchema->colId = htonl(pSchema->colId);
-    pSchema++;
-  }
-
-  pSchema = pMetaMsg->pSchema;
   tfree(pRequest->body.resInfo.pRspMsg);
-
   pRequest->body.resInfo.pRspMsg = pMsg->pData;
   SReqResultInfo* pResInfo = &pRequest->body.resInfo;
 
   if (pResInfo->fields == NULL) {
     TAOS_FIELD* pFields = calloc(pMetaMsg->numOfColumns, sizeof(TAOS_FIELD));
     for (int32_t i = 0; i < pMetaMsg->numOfColumns; ++i) {
-      tstrncpy(pFields[i].name, pSchema[i].name, tListLen(pFields[i].name));
-      pFields[i].type = pSchema[i].type;
-      pFields[i].bytes = pSchema[i].bytes;
+      SSchema* pSchema = &pMetaMsg->pSchemas[i];
+      tstrncpy(pFields[i].name, pSchema->name, tListLen(pFields[i].name));
+      pFields[i].type = pSchema->type;
+      pFields[i].bytes = pSchema->bytes;
     }
 
     pResInfo->fields = pFields;
   }
 
   pResInfo->numOfCols = pMetaMsg->numOfColumns;
-  pRequest->body.showInfo.execId = pShow->showId;
+  pRequest->body.showInfo.execId = showRsp.showId;
+  tFreeSShowRsp(&showRsp);
 
   // todo
   if (pRequest->type == TDMT_VND_SHOW_TABLES) {

source/client/test/CMakeLists.txt

@@ -8,13 +8,13 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 ADD_EXECUTABLE(clientTest clientTests.cpp)
 TARGET_LINK_LIBRARIES(
         clientTest
-        PUBLIC os util common transport parser catalog scheduler function gtest taos qcom
+        PUBLIC os util common transport parser catalog scheduler function gtest taos qcom config
 )
 
 ADD_EXECUTABLE(tmqTest tmqTest.cpp)
 TARGET_LINK_LIBRARIES(
         tmqTest
-        PUBLIC os util common transport parser catalog scheduler function gtest taos qcom
+        PUBLIC os util common transport parser catalog scheduler function gtest taos qcom config
 )
 
 TARGET_INCLUDE_DIRECTORIES(

source/client/test/clientTests.cpp

@@ -49,7 +49,7 @@ int main(int argc, char** argv) {
 }
 
 TEST(testCase, driverInit_Test) {
-  taosInitGlobalCfg();
+  // taosInitGlobalCfg();
 //  taos_init();
 }
 

source/client/test/tmqTest.cpp

@@ -34,7 +34,7 @@ int main(int argc, char** argv) {
 }
 
 TEST(testCase, driverInit_Test) {
-  taosInitGlobalCfg();
+  // taosInitGlobalCfg();
 //  taos_init();
 }
 

source/common/src/tcompare.c

@@ -15,116 +15,6 @@
 
 #include "tcompare.h"
 
-int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
-  SPatternCompareInfo pInfo = {'%', '_'};
-
-  assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN);
-  char *pattern = calloc(varDataLen(pRight) + 1, sizeof(char));
-  memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
-
-  size_t sz = varDataLen(pLeft);
-  char *buf = malloc(sz + 1);
-  memcpy(buf, varDataVal(pLeft), sz);
-  buf[sz] = 0;
-
-  int32_t ret = patternMatch(pattern, buf, sz, &pInfo);
-  free(buf);
-  free(pattern);
-  return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
-}
-
-int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
-  SPatternCompareInfo pInfo = {'%', '_'};
-
-  assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE);
-
-  wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t));
-  memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
-
-  int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft)/TSDB_NCHAR_SIZE, &pInfo);
-  free(pattern);
-
-  return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
-}
-
-__compar_fn_t getComparFunc(int32_t type, int32_t optr) {
-  __compar_fn_t comparFn = NULL;
-
-  if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) {
-    switch (type) {
-      case TSDB_DATA_TYPE_BOOL:
-      case TSDB_DATA_TYPE_TINYINT:
-      case TSDB_DATA_TYPE_UTINYINT:
-        return setCompareBytes1;
-      case TSDB_DATA_TYPE_SMALLINT:
-      case TSDB_DATA_TYPE_USMALLINT:
-        return setCompareBytes2;
-      case TSDB_DATA_TYPE_INT:
-      case TSDB_DATA_TYPE_UINT:
-      case TSDB_DATA_TYPE_FLOAT:
-        return setCompareBytes4;
-      case TSDB_DATA_TYPE_BIGINT:
-      case TSDB_DATA_TYPE_UBIGINT:
-      case TSDB_DATA_TYPE_DOUBLE:
-      case TSDB_DATA_TYPE_TIMESTAMP:
-        return setCompareBytes8;
-      default:
-        assert(0);
-    }
-  }
-
-  switch (type) {
-    case TSDB_DATA_TYPE_BOOL:
-    case TSDB_DATA_TYPE_TINYINT:   comparFn = compareInt8Val;   break;
-    case TSDB_DATA_TYPE_SMALLINT:  comparFn = compareInt16Val;  break;
-    case TSDB_DATA_TYPE_INT:       comparFn = compareInt32Val;  break;
-    case TSDB_DATA_TYPE_BIGINT:
-    case TSDB_DATA_TYPE_TIMESTAMP: comparFn = compareInt64Val;  break;
-    case TSDB_DATA_TYPE_FLOAT:     comparFn = compareFloatVal;  break;
-    case TSDB_DATA_TYPE_DOUBLE:    comparFn = compareDoubleVal; break;
-    case TSDB_DATA_TYPE_BINARY: {
-      if (optr == TSDB_RELATION_MATCH) {
-        comparFn = compareStrRegexCompMatch;
-      } else if (optr == TSDB_RELATION_NMATCH) {
-        comparFn = compareStrRegexCompNMatch;
-      } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */
-        comparFn = compareStrPatternComp;
-      } else if (optr == TSDB_RELATION_IN) {
-        comparFn = compareFindItemInSet;
-      } else { /* normal relational comparFn */
-        comparFn = compareLenPrefixedStr;
-      }
-
-      break;
-    }
-
-    case TSDB_DATA_TYPE_NCHAR: {
-      if (optr == TSDB_RELATION_MATCH) {
-        comparFn = compareStrRegexCompMatch;
-      } else if (optr == TSDB_RELATION_NMATCH) {
-        comparFn = compareStrRegexCompNMatch;
-      } else if (optr == TSDB_RELATION_LIKE) {
-        comparFn = compareWStrPatternComp;
-      } else if (optr == TSDB_RELATION_IN) {
-        comparFn = compareFindItemInSet;
-      } else {
-        comparFn = compareLenPrefixedWStr;
-      }
-      break;
-    }
-
-    case TSDB_DATA_TYPE_UTINYINT:  comparFn = compareUint8Val; break;
-    case TSDB_DATA_TYPE_USMALLINT: comparFn = compareUint16Val;break;
-    case TSDB_DATA_TYPE_UINT:      comparFn = compareUint32Val;break;
-    case TSDB_DATA_TYPE_UBIGINT:   comparFn = compareUint64Val;break;
-
-    default:
-      comparFn = compareInt32Val;
-      break;
-  }
-
-  return comparFn;
-}
 
 __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) {
   __compar_fn_t comparFn = NULL;
@@ -218,4 +108,4 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) {
       }
     }
   }
-}
+}
\ No newline at end of file

source/common/src/tname.c

@@ -31,7 +31,7 @@ SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFil
   }
 
   assert(src->filterstr == 0 || src->filterstr == 1);
-  assert(!(src->lowerRelOptr == TSDB_RELATION_INVALID && src->upperRelOptr == TSDB_RELATION_INVALID));
+  assert(!(src->lowerRelOptr == 0 && src->upperRelOptr == 0));
 
   return pFilter;
 }