merge 3.0

contrib/CMakeLists.txt

@@ -193,6 +193,7 @@ endif(${BUILD_WITH_TRAFT})
 
 # LIBUV
 if(${BUILD_WITH_UV})
+    add_compile_options(-Wno-sign-compare)
     add_subdirectory(libuv)
 endif(${BUILD_WITH_UV})
 

example/src/tmq.c

@@ -66,7 +66,7 @@ int32_t init_env() {
 }
 
 int32_t create_topic() {
-  printf("create topic");
+  printf("create topic\n");
   TAOS_RES* pRes;
   TAOS*     pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   if (pConn == NULL) {
@@ -91,6 +91,10 @@ int32_t create_topic() {
   return 0;
 }
 
+void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) {
+  printf("commit %d\n", resp);
+}
+
 tmq_t* build_consumer() {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   assert(pConn != NULL);
@@ -103,6 +107,7 @@ tmq_t* build_consumer() {
 
   tmq_conf_t* conf = tmq_conf_new();
   tmq_conf_set(conf, "group.id", "tg2");
+  tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print);
   tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0);
   return tmq;
 }
@@ -144,7 +149,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
 }
 
 void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
-  static const int MIN_COMMIT_COUNT = 1000;
+  static const int MIN_COMMIT_COUNT = 1;
 
   int            msg_count = 0;
   tmq_resp_err_t err;
@@ -214,6 +219,6 @@ int main(int argc, char* argv[]) {
   tmq_t*      tmq = build_consumer();
   tmq_list_t* topic_list = build_topic_list();
   /*perf_loop(tmq, topic_list);*/
-  basic_consume_loop(tmq, topic_list);
-  /*sync_consume_loop(tmq, topic_list);*/
+  /*basic_consume_loop(tmq, topic_list);*/
+  sync_consume_loop(tmq, topic_list);
 }

include/client/taos.h

@@ -198,8 +198,8 @@ DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLi
 /* --------------------------TMQ INTERFACE------------------------------- */
 
 enum tmq_resp_err_t {
+  TMQ_RESP_ERR__FAIL = -1,
   TMQ_RESP_ERR__SUCCESS = 0,
-  TMQ_RESP_ERR__FAIL = 1,
 };
 
 typedef enum tmq_resp_err_t tmq_resp_err_t;
@@ -226,7 +226,7 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
 DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list);
 #if 0
 DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq);
-DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_topic_vgroup_list_t** topics);
+DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics);
 #endif
 DLL_EXPORT tmq_message_t *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time);
 DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq);
@@ -238,6 +238,7 @@ DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *
 #if 0
 DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async);
 #endif
+DLL_EXPORT tmq_resp_err_t tmq_seek(tmq_t *tmq, const tmq_topic_vgroup_t *offset);
 /* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */
 
 enum tmq_conf_res_t {

include/common/common.h

@@ -16,7 +16,6 @@
 #ifndef TDENGINE_COMMON_H
 #define TDENGINE_COMMON_H
 
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -43,14 +42,16 @@ extern "C" {
 //   int16_t bytes;
 // } SSchema;
 
-#define TMQ_REQ_TYPE_COMMIT_ONLY 0
-#define TMQ_REQ_TYPE_CONSUME_ONLY 1
-#define TMQ_REQ_TYPE_CONSUME_AND_COMMIT 2
+enum {
+  TMQ_CONF__RESET_OFFSET__LATEST = -1,
+  TMQ_CONF__RESET_OFFSET__EARLIEAST = -2,
+  TMQ_CONF__RESET_OFFSET__NONE = -3,
+};
 
 typedef struct {
   uint32_t  numOfTables;
-  SArray   *pGroupList;
-  SHashObj *map;  // speedup acquire the tableQueryInfo by table uid
+  SArray*   pGroupList;
+  SHashObj* map;  // speedup acquire the tableQueryInfo by table uid
 } STableGroupInfo;
 
 typedef struct SColumnDataAgg {
@@ -79,14 +80,14 @@ typedef struct SConstantItem {
 
 // info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList);
 typedef struct SSDataBlock {
-  SColumnDataAgg *pBlockAgg;
-  SArray         *pDataBlock;    // SArray<SColumnInfoData>
-  SArray         *pConstantList; // SArray<SConstantItem>, it is a constant/tags value of the corresponding result value.
+  SColumnDataAgg* pBlockAgg;
+  SArray*         pDataBlock;  // SArray<SColumnInfoData>
+  SArray* pConstantList;       // SArray<SConstantItem>, it is a constant/tags value of the corresponding result value.
   SDataBlockInfo info;
 } SSDataBlock;
 
 typedef struct SVarColAttr {
-  int32_t *offset;    // start position for each entry in the list
+  int32_t* offset;    // start position for each entry in the list
   uint32_t length;    // used buffer size that contain the valid data
   uint32_t allocLen;  // allocated buffer size
 } SVarColAttr;
@@ -95,10 +96,10 @@ typedef struct SVarColAttr {
 // pBlockAgg->numOfNull == 0, no data are null.
 typedef struct SColumnInfoData {
   SColumnInfo info;     // TODO filter info needs to be removed
-  bool        hasNull;// if current column data has null value.
-  char       *pData;  // the corresponding block data in memory
+  bool        hasNull;  // if current column data has null value.
+  char*       pData;    // the corresponding block data in memory
   union {
-    char       *nullbitmap;  // bitmap, one bit for each item in the list
+    char*       nullbitmap;  // bitmap, one bit for each item in the list
     SVarColAttr varmeta;
   };
 } SColumnInfoData;
@@ -149,7 +150,6 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp
   int32_t tlen = 0;
   int32_t sz = 0;
   tlen += taosEncodeFixedI64(buf, pRsp->consumerId);
-  tlen += taosEncodeFixedI64(buf, pRsp->committedOffset);
   tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
   tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
   tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum);
@@ -170,7 +170,6 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp
 static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) {
   int32_t sz;
   buf = taosDecodeFixedI64(buf, &pRsp->consumerId);
-  buf = taosDecodeFixedI64(buf, &pRsp->committedOffset);
   buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
   buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
   buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);

include/common/tep.h

@@ -23,7 +23,7 @@ typedef struct SBlockOrderInfo {
 //  bool             hasNull;
 } SBlockOrderInfo;
 
-int  taosGetFqdnPortFromEp(const char *ep, SEp *pEp);
+int  taosGetFqdnPortFromEp(const char *ep, uint16_t defaultPort, SEp *pEp);
 void addEpIntoEpSet(SEpSet *pEpSet, const char *fqdn, uint16_t port);
 
 bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2);
@@ -68,8 +68,8 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
 }
 
 #define colDataGet(p1_, r_)                                                          \
-  ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? (p1_)->pData + (p1_)->varmeta.offset[(r_)] \
-                                        : (p1_)->pData + ((r_) * (p1_)->info.bytes));
+  ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) \
+                                        : ((p1_)->pData + ((r_) * (p1_)->info.bytes)))
 
 int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull);
 int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, uint32_t numOfRow2);

include/common/tglobal.h

@@ -21,35 +21,11 @@ extern "C" {
 #endif
 
 #include "tdef.h"
-#include "tcfg.h"
-
-// cluster
-extern char     tsFirst[];
-extern char     tsSecond[];
-extern char     tsLocalFqdn[];
-extern char     tsLocalEp[];
-extern uint16_t tsServerPort;
-extern int32_t  tsStatusInterval;
-extern int8_t   tsEnableTelemetryReporting;
-extern int32_t  tsNumOfSupportVnodes;
 
 // common
-extern int      tsRpcTimer;
-extern int      tsRpcMaxTime;
-extern int      tsRpcForceTcp;  // all commands go to tcp protocol if this is enabled
-extern int32_t  tsMaxConnections;
-extern int32_t  tsMaxShellConns;
-extern int32_t  tsShellActivityTimer;
-extern uint32_t tsMaxTmrCtrl;
-extern float    tsNumOfThreadsPerCore;
-extern int32_t  tsNumOfCommitThreads;
-extern float    tsRatioOfQueryCores;
-extern int8_t   tsDaylight;
-extern int8_t   tsEnableCoreFile;
 extern int32_t  tsCompressMsgSize;
 extern int32_t  tsCompressColData;
 extern int32_t  tsMaxNumOfDistinctResults;
-extern char     tsTempDir[];
 extern int      tsCompatibleModel;  // 2.0 compatible model
 extern int8_t   tsEnableSlaveQuery;
 extern int8_t   tsEnableAdjustMaster;
@@ -66,7 +42,6 @@ extern int8_t  tsDeadLockKillQuery;
 // client
 extern int32_t tsMaxWildCardsLen;
 extern int32_t tsMaxRegexStringLen;
-extern int8_t  tsTscEnableRecordSql;
 extern int32_t tsMaxNumOfOrderedResults;
 extern int32_t tsMinSlidingTime;
 extern int32_t tsMinIntervalTime;
@@ -77,41 +52,8 @@ extern float   tsStreamComputDelayRatio;  // the delayed computing ration of the
 extern int32_t tsProjectExecInterval;
 extern int64_t tsMaxRetentWindow;
 
-// system info
-extern float    tsTotalLogDirGB;
-extern float    tsTotalTmpDirGB;
-extern float    tsTotalDataDirGB;
-extern float    tsAvailLogDirGB;
-extern float    tsAvailTmpDirectorySpace;
-extern float    tsAvailDataDirGB;
-extern float    tsUsedDataDirGB;
-extern float    tsMinimalLogDirGB;
-extern float    tsReservedTmpDirectorySpace;
-extern float    tsMinimalDataDirGB;
-extern uint32_t tsVersion;
-
-// build info
-extern char version[];
-extern char compatible_version[];
-extern char gitinfo[];
-extern char gitinfoOfInternal[];
-extern char buildinfo[];
-
-// lossy
-extern char     tsLossyColumns[];
-extern double   tsFPrecision;
-extern double   tsDPrecision;
-extern uint32_t tsMaxRange;
-extern uint32_t tsCurRange;
-extern char     tsCompressor[];
-
-extern int32_t  tsDiskCfgNum;
-extern SDiskCfg tsDiskCfg[];
-
 #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
 
-void    taosInitGlobalCfg();
-int32_t taosCheckAndPrintCfg();
 int32_t taosCfgDynamicOptions(char *msg);
 bool    taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId);
 void    taosAddDataDir(int index, char *v1, int level, int primary);

include/common/tmsg.h

@@ -422,7 +422,7 @@ typedef struct {
 } SColumnInfo;
 
 typedef struct {
-  uint64_t uid;
+  int64_t uid;
   TSKEY   key;  // last accessed ts, for subscription
 } STableIdInfo;
 
@@ -555,7 +555,7 @@ int32_t tDeserializeSDropDbReq(void* buf, int32_t bufLen, SDropDbReq* pReq);
 
 typedef struct {
   char    db[TSDB_DB_FNAME_LEN];
-  uint64_t uid;
+  int64_t uid;
 } SDropDbRsp;
 
 int32_t tSerializeSDropDbRsp(void* buf, int32_t bufLen, SDropDbRsp* pRsp);
@@ -571,7 +571,7 @@ int32_t tDeserializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq);
 
 typedef struct {
   char    db[TSDB_DB_FNAME_LEN];
-  uint64_t uid;
+  int64_t uid;
   int32_t vgVersion;
   int32_t vgNum;
   int8_t  hashMethod;
@@ -656,9 +656,9 @@ int32_t tDeserializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp
 typedef struct {
   int32_t statusInterval;
   int64_t checkTime;                    // 1970-01-01 00:00:00.000
-  char    timezone[TSDB_TIMEZONE_LEN];  // tsTimezone
-  char    locale[TSDB_LOCALE_LEN];      // tsLocale
-  char    charset[TSDB_LOCALE_LEN];     // tsCharset
+  char    timezone[TD_TIMEZONE_LEN];  // tsTimezone
+  char    locale[TD_LOCALE_LEN];      // tsLocale
+  char    charset[TD_LOCALE_LEN];     // tsCharset
 } SClusterCfg;
 
 typedef struct {
@@ -725,7 +725,7 @@ typedef struct {
   int32_t  vgId;
   int32_t  dnodeId;
   char     db[TSDB_DB_FNAME_LEN];
-  uint64_t dbUid;
+  int64_t  dbUid;
   int32_t  vgVersion;
   int32_t  cacheBlockSize;
   int32_t  totalBlocks;
@@ -755,7 +755,7 @@ int32_t tDeserializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq*
 typedef struct {
   int32_t vgId;
   int32_t dnodeId;
-  uint64_t dbUid;
+  int64_t dbUid;
   char    db[TSDB_DB_FNAME_LEN];
 } SDropVnodeReq, SSyncVnodeReq, SCompactVnodeReq;
 
@@ -796,7 +796,7 @@ typedef struct {
   char     tbName[TSDB_TABLE_NAME_LEN];
   char     stbName[TSDB_TABLE_NAME_LEN];
   char     dbFName[TSDB_DB_FNAME_LEN];
-  uint64_t dbId;
+  int64_t  dbId;
   int32_t  numOfTags;
   int32_t  numOfColumns;
   int8_t   precision;
@@ -1268,7 +1268,7 @@ typedef struct {
 typedef struct {
   SMsgHead head;
   char     name[TSDB_TABLE_FNAME_LEN];
-  uint64_t tuid;
+  int64_t  tuid;
   int32_t  sverson;
   int32_t  execLen;
   char*    executor;
@@ -1279,11 +1279,11 @@ typedef struct {
 typedef struct {
   SMsgHead head;
   char     name[TSDB_TABLE_FNAME_LEN];
-  uint64_t tuid;
+  int64_t  tuid;
 } SDDropTopicReq;
 
 typedef struct SVCreateTbReq {
-  uint64_t ver;  // use a general definition
+  int64_t  ver;  // use a general definition
   char*    name;
   uint32_t ttl;
   uint32_t keep;
@@ -1314,7 +1314,7 @@ int32_t tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq);
 void*   tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq);
 
 typedef struct {
-  uint64_t ver;  // use a general definition
+  int64_t ver;  // use a general definition
   SArray* pArray;
 } SVCreateTbBatchReq;
 
@@ -1325,7 +1325,7 @@ int32_t tSerializeSVCreateTbBatchReq(void** buf, SVCreateTbBatchReq* pReq);
 void*   tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pReq);
 
 typedef struct {
-  uint64_t ver;
+  int64_t  ver;
   char*    name;
   uint8_t  type;
   tb_uid_t suid;
@@ -1760,35 +1760,19 @@ typedef struct {
   char    cgroup[TSDB_CONSUMER_GROUP_LEN];
 } SMqOffset;
 
-typedef struct {
-  int32_t vgId;
-  SArray* offsets;  // SArray<SMqOffset>
-} SMqVgOffsets;
-
 typedef struct {
   int32_t    num;
   SMqOffset* offsets;
-} SMqCMResetOffsetReq;
-
-typedef struct {
-  int32_t reserved;
-} SMqCMResetOffsetRsp;
-
-typedef struct {
-  int64_t      leftForVer;
-  SMqVgOffsets offsets;
-} SMqMVResetOffsetReq;
+} SMqCMCommitOffsetReq;
 
 typedef struct {
   int32_t reserved;
-} SMqMVResetOffsetRsp;
+} SMqCMCommitOffsetRsp;
 
 int32_t tEncodeSMqOffset(SCoder* encoder, const SMqOffset* pOffset);
 int32_t tDecodeSMqOffset(SCoder* decoder, SMqOffset* pOffset);
-int32_t tEncodeSMqCMResetOffsetReq(SCoder* encoder, const SMqCMResetOffsetReq* pReq);
-int32_t tDecodeSMqCMResetOffsetReq(SCoder* decoder, SMqCMResetOffsetReq* pReq);
-int32_t tEncodeSMqMVResetOffsetReq(SCoder* encoder, const SMqMVResetOffsetReq* pReq);
-int32_t tDecodeSMqMVResetOffsetReq(SCoder* decoder, SMqMVResetOffsetReq* pReq);
+int32_t tEncodeSMqCMCommitOffsetReq(SCoder* encoder, const SMqCMCommitOffsetReq* pReq);
+int32_t tDecodeSMqCMCommitOffsetReq(SCoder* decoder, SMqCMCommitOffsetReq* pReq);
 
 typedef struct {
   uint32_t nCols;
@@ -1870,7 +1854,6 @@ typedef struct {
 typedef struct {
   int64_t         consumerId;
   SSchemaWrapper* schemas;
-  int64_t         committedOffset;
   int64_t         reqOffset;
   int64_t         rspOffset;
   int32_t         skipLogNum;
@@ -1881,22 +1864,18 @@ typedef struct {
 // one req for one vg+topic
 typedef struct {
   SMsgHead head;
-  // 0: commit only, current offset
-  // 1: consume only, poll next offset
-  // 2: commit current and consume next offset
-  int32_t reqType;
 
-  int64_t reqId;
   int64_t consumerId;
   int64_t blockingTime;
   char    cgroup[TSDB_CONSUMER_GROUP_LEN];
 
-  int64_t offset;
+  int64_t currentOffset;
   char    topic[TSDB_TOPIC_FNAME_LEN];
 } SMqConsumeReq;
 
 typedef struct {
   int32_t vgId;
+  int64_t offset;
   SEpSet  epSet;
 } SMqSubVgEp;
 
@@ -1917,12 +1896,14 @@ static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taos
 static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) {
   int32_t tlen = 0;
   tlen += taosEncodeFixedI32(buf, pVgEp->vgId);
+  tlen += taosEncodeFixedI64(buf, pVgEp->offset);
   tlen += taosEncodeSEpSet(buf, &pVgEp->epSet);
   return tlen;
 }
 
 static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) {
   buf = taosDecodeFixedI32(buf, &pVgEp->vgId);
+  buf = taosDecodeFixedI64(buf, &pVgEp->offset);
   buf = taosDecodeSEpSet(buf, &pVgEp->epSet);
   return buf;
 }

include/common/tmsgdef.h

@@ -135,18 +135,19 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_SHOW, "mnode-show", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_SHOW_RETRIEVE, "mnode-retrieve", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_STATUS, "mnode-status", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_MND_TRANS, "mnode-trans", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "mnode-trans-tmr", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_KILL_TRANS, "mnode-kill-trans", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_TELEM_TIMER, "mnode-telem-tmr", SMTimerReq, SMTimerReq)
   TD_DEF_MSG_TYPE(TDMT_MND_GRANT, "mnode-grant", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_AUTH, "mnode-auth", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "mnode-create-topic", SMCreateTopicReq, SMCreateTopicRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp)
-  TD_DEF_MSG_TYPE(TDMT_MND_RESET_OFFSET, "mnode-reset-offset", SMqCMResetOffsetReq, SMqCMResetOffsetRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp)
-  TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-timer", SMTimerReq, SMTimerReq)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp)
 
   // Requests handled by VNODE
   TD_NEW_MSG_SEG(TDMT_VND_MSG)
@@ -194,4 +195,3 @@ enum {
   TDMT_MAX
 #endif
 };
-// clang-format on

include/common/ttypes.h

@@ -91,6 +91,8 @@ typedef struct {
   do {                                         \
     switch (_type) {                           \
       case TSDB_DATA_TYPE_BOOL:                \
+        *(bool *)(_v) = (bool)(_data);     \
+        break;                                 \
       case TSDB_DATA_TYPE_TINYINT:             \
         *(int8_t *)(_v) = (int8_t)(_data);     \
         break;                                 \
@@ -104,6 +106,7 @@ typedef struct {
         *(uint16_t *)(_v) = (uint16_t)(_data); \
         break;                                 \
       case TSDB_DATA_TYPE_BIGINT:              \
+      case TSDB_DATA_TYPE_TIMESTAMP:           \
         *(int64_t *)(_v) = (int64_t)(_data);   \
         break;                                 \
       case TSDB_DATA_TYPE_UBIGINT:             \

include/dnode/mgmt/dnode.h

@@ -29,11 +29,14 @@ typedef struct SDnode SDnode;
 typedef struct {
   int32_t  sver;
   int32_t  numOfCores;
-  int16_t numOfCommitThreads;
-  int8_t  enableTelem;
-  char    timezone[TSDB_TIMEZONE_LEN];
-  char    locale[TSDB_LOCALE_LEN];
-  char    charset[TSDB_LOCALE_LEN];
+  uint16_t numOfCommitThreads;
+  bool     enableTelem;
+  bool     printAuth;
+  int32_t  rpcTimer;
+  int32_t  rpcMaxTime;
+  char     timezone[TD_TIMEZONE_LEN];
+  char     locale[TD_LOCALE_LEN];
+  char     charset[TD_LOCALE_LEN];
   char     buildinfo[64];
   char     gitinfo[48];
 } SDnodeEnvCfg;
@@ -65,6 +68,7 @@ typedef struct {
   char     localEp[TSDB_EP_LEN];
   char     localFqdn[TSDB_FQDN_LEN];
   char     firstEp[TSDB_EP_LEN];
+  char     secondEp[TSDB_EP_LEN];
 } SDnodeObjCfg;
 
 /**

include/dnode/mnode/mnode.h

@@ -46,7 +46,8 @@ typedef struct SMnodeLoad {
 
 typedef struct SMnodeCfg {
   int32_t sver;
-  int8_t  enableTelem;
+  bool    enableTelem;
+  bool    printAuth;
   int32_t statusInterval;
   int32_t shellActivityTimer;
   char   *timezone;

include/dnode/mnode/sdb/sdb.h

@@ -113,14 +113,15 @@ typedef enum {
   SDB_USER = 7,
   SDB_AUTH = 8,
   SDB_ACCT = 9,
-  SDB_SUBSCRIBE = 10,
-  SDB_CONSUMER = 11,
-  SDB_TOPIC = 12,
-  SDB_VGROUP = 13,
-  SDB_STB = 14,
-  SDB_DB = 15,
-  SDB_FUNC = 16,
-  SDB_MAX = 17
+  SDB_OFFSET = 10,
+  SDB_SUBSCRIBE = 11,
+  SDB_CONSUMER = 12,
+  SDB_TOPIC = 13,
+  SDB_VGROUP = 14,
+  SDB_STB = 15,
+  SDB_DB = 16,
+  SDB_FUNC = 17,
+  SDB_MAX = 18
 } ESdbType;
 
 typedef struct SSdb SSdb;

include/libs/config/config.h

+
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_CONFIG_H_
+#define _TD_CONFIG_H_
+
+#include "os.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define CFG_NAME_MAX_LEN 128
+
+typedef enum {
+  CFG_STYPE_DEFAULT,
+  CFG_STYPE_CFG_FILE,
+  CFG_STYPE_ENV_FILE,
+  CFG_STYPE_ENV_VAR,
+  CFG_STYPE_APOLLO_URL,
+  CFG_STYPE_ARG_LIST,
+  CFG_STYPE_API_OPTION
+} ECfgSrcType;
+
+typedef enum {
+  CFG_DTYPE_NONE,
+  CFG_DTYPE_BOOL,
+  CFG_DTYPE_INT32,
+  CFG_DTYPE_INT64,
+  CFG_DTYPE_FLOAT,
+  CFG_DTYPE_STRING,
+  CFG_DTYPE_IPSTR,
+  CFG_DTYPE_DIR,
+  CFG_DTYPE_LOCALE,
+  CFG_DTYPE_CHARSET,
+  CFG_DTYPE_TIMEZONE
+} ECfgDataType;
+
+typedef struct SConfigItem {
+  ECfgSrcType  stype;
+  ECfgDataType dtype;
+  char        *name;
+  union {
+    bool     bval;
+    float    fval;
+    int32_t  i32;
+    int64_t  i64;
+    char    *str;
+  };
+  union {
+    int64_t imin;
+    double  fmin;
+  };
+  union {
+    int64_t imax;
+    double  fmax;
+  };
+} SConfigItem;
+
+typedef struct SConfig SConfig;
+
+SConfig *cfgInit();
+int32_t  cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr);
+void     cfgCleanup(SConfig *pCfg);
+
+int32_t      cfgGetSize(SConfig *pCfg);
+SConfigItem *cfgIterate(SConfig *pCfg, SConfigItem *pIter);
+void         cfgCancelIterate(SConfig *pCfg, SConfigItem *pIter);
+SConfigItem *cfgGetItem(SConfig *pCfg, const char *name);
+int32_t      cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype);
+
+int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal);
+int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval);
+int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval);
+int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval);
+int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal);
+int32_t cfgAddIpStr(SConfig *pCfg, const char *name, const char *defaultVa);
+int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal);
+int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal);
+int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal);
+int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal);
+
+const char *cfgStypeStr(ECfgSrcType type);
+const char *cfgDtypeStr(ECfgDataType type);
+
+void cfgDumpCfg(SConfig *pCfg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_CONFIG_H_*/

include/libs/function/function.h

@@ -228,6 +228,7 @@ typedef struct SAggFunctionInfo {
 
 typedef struct SScalarParam {
   void*   data;
+  bool    colData;
   int32_t num;
   int32_t type;
   int32_t bytes;

include/libs/nodes/querynodes.h

@@ -90,45 +90,6 @@ typedef struct SValueNode {
   } datum;
 } SValueNode;
 
-typedef enum EOperatorType {
-  // arithmetic operator
-  OP_TYPE_ADD = 1,
-  OP_TYPE_SUB,
-  OP_TYPE_MULTI,
-  OP_TYPE_DIV,
-  OP_TYPE_MOD,
-
-  // bit operator
-  OP_TYPE_BIT_AND,
-  OP_TYPE_BIT_OR,
-
-  // comparison operator
-  OP_TYPE_GREATER_THAN,
-  OP_TYPE_GREATER_EQUAL,
-  OP_TYPE_LOWER_THAN,
-  OP_TYPE_LOWER_EQUAL,
-  OP_TYPE_EQUAL,
-  OP_TYPE_NOT_EQUAL,
-  OP_TYPE_IN,
-  OP_TYPE_NOT_IN,
-  OP_TYPE_LIKE,
-  OP_TYPE_NOT_LIKE,
-  OP_TYPE_MATCH,
-  OP_TYPE_NMATCH,
-  OP_TYPE_IS_NULL,
-  OP_TYPE_IS_NOT_NULL,
-  OP_TYPE_IS_TRUE,
-  OP_TYPE_IS_FALSE,
-  OP_TYPE_IS_UNKNOWN,
-  OP_TYPE_IS_NOT_TRUE,
-  OP_TYPE_IS_NOT_FALSE,
-  OP_TYPE_IS_NOT_UNKNOWN,
-
-  // json operator
-  OP_TYPE_JSON_GET_VALUE,
-  OP_TYPE_JSON_CONTAINS
-} EOperatorType;
-
 typedef struct SOperatorNode {
   SExprNode node; // QUERY_NODE_OPERATOR
   EOperatorType opType;
@@ -136,11 +97,6 @@ typedef struct SOperatorNode {
   SNode* pRight;
 } SOperatorNode;
 
-typedef enum ELogicConditionType {
-  LOGIC_COND_TYPE_AND,
-  LOGIC_COND_TYPE_OR,
-  LOGIC_COND_TYPE_NOT,
-} ELogicConditionType;
 
 typedef struct SLogicConditionNode {
   SExprNode node; // QUERY_NODE_LOGIC_CONDITION

include/libs/scalar/filter.h

@@ -20,6 +20,14 @@ extern "C" {
 #endif
 
 typedef struct SFilterInfo SFilterInfo;
+typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **);
+
+
+enum {
+  FLT_OPTION_NO_REWRITE = 1,
+  FLT_OPTION_TIMESTAMP = 2,
+  FLT_OPTION_NEED_UNIQE = 4,
+};
 
 typedef struct SFilterColumnParam{
   int32_t numOfCols;
@@ -27,6 +35,16 @@ typedef struct SFilterColumnParam{
 } SFilterColumnParam;
 
 
+extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t options);
+extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols);
+extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param);
+extern int32_t filterSetDataFromColId(SFilterInfo *info, void *param);
+extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win);
+extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar);
+extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo);
+extern void filterFreeInfo(SFilterInfo *info);
+extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows);
+
 #ifdef __cplusplus
 }
 #endif

include/libs/scalar/scalar.h

@@ -29,6 +29,7 @@ typedef struct SFilterInfo SFilterInfo;
 int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes);
 int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst);
 int32_t scalarGetOperatorParamNum(EOperatorType type);
+int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type);
 
 int32_t vectorGetConvertType(int32_t type1, int32_t type2);
 int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut);

include/libs/transport/trpc.h

@@ -78,12 +78,21 @@ typedef struct SRpcInit {
   // call back to retrieve the client auth info, for server app only
   int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
 
+  // call back to keep conn or not
+  bool (*pfp)(void *parent, tmsg_t msgType);
+
   void *parent;
 } SRpcInit;
 
-int32_t rpcInit();
+typedef struct {
+  int32_t rpcTimer;
+  int32_t rpcMaxTime;
+  int32_t sver;
+} SRpcCfg;
+
+int32_t rpcInit(SRpcCfg *pCfg);
 void    rpcCleanup();
-void *  rpcOpen(const SRpcInit *pRpc);
+void   *rpcOpen(const SRpcInit *pRpc);
 void    rpcClose(void *);
 void *  rpcMallocCont(int contLen);
 void    rpcFreeCont(void *pCont);

include/os/os.h

@@ -59,6 +59,7 @@ extern "C" {
 #include "osEndian.h"
 #include "osEnv.h"
 #include "osFile.h"
+#include "osLocale.h"
 #include "osLz4.h"
 #include "osMath.h"
 #include "osMemory.h"
@@ -73,6 +74,7 @@ extern "C" {
 #include "osThread.h"
 #include "osTime.h"
 #include "osTimer.h"
+#include "osTimezone.h"
 
 void osInit();
 

include/os/osDir.h

@@ -23,8 +23,8 @@ extern "C" {
 void    taosRemoveDir(const char *dirname);
 int32_t taosDirExist(char *dirname);
 int32_t taosMkDir(const char *dirname);
-void    taosRemoveOldFiles(char *dirname, int32_t keepDays);
-int32_t taosExpandDir(char *dirname, char *outname, int32_t maxlen);
+void    taosRemoveOldFiles(const char *dirname, int32_t keepDays);
+int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen);
 int32_t taosRealPath(char *dirname, int32_t maxlen);
 
 #ifdef __cplusplus

include/os/osEnv.h

@@ -16,16 +16,38 @@
 #ifndef _TD_OS_ENV_H_
 #define _TD_OS_ENV_H_
 
+#include "osSysinfo.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-extern char tsOsName[];
-extern char tsDataDir[];
-extern char tsLogDir[];
-extern char tsScriptDir[];
+typedef struct SOsEnv SOsEnv;
+
 extern char configDir[];
 
+void osInit();
+void osUpdate();
+
+bool   osLogSpaceAvailable();
+int8_t osDaylight();
+
+const char *osLogDir();
+const char *osTempDir();
+const char *osDataDir();
+const char *osName();
+const char *osTimezone();
+const char *osLocale();
+const char *osCharset();
+
+void osSetLogDir(const char *logDir);
+void osSetTempDir(const char *tempDir);
+void osSetDataDir(const char *dataDir);
+void osSetLogReservedSpace(float sizeInGB);
+void osSetTempReservedSpace(float sizeInGB);
+void osSetDataReservedSpace(float sizeInGB);
+void osSetTimezone(const char *timezone);
+
 #ifdef __cplusplus
 }
 #endif

source/libs/scalar/test/scalarTests.cpp → include/os/osLocale.h

@@ -13,43 +13,22 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <gtest/gtest.h>
-#include <tglobal.h>
-#include <iostream>
-
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wwrite-strings"
-#pragma GCC diagnostic ignored "-Wunused-function"
-#pragma GCC diagnostic ignored "-Wunused-variable"
-#pragma GCC diagnostic ignored "-Wsign-compare"
-#pragma GCC diagnostic ignored "-Wsign-compare"
-#pragma GCC diagnostic ignored "-Wformat"
-#pragma GCC diagnostic ignored "-Wint-to-pointer-cast"
-#pragma GCC diagnostic ignored "-Wpointer-arith"
+#ifndef _TD_OS_LOCALE_H_
+#define _TD_OS_LOCALE_H_
 
 #include "os.h"
+#include "osString.h"
 
-#include "taos.h"
-#include "tdef.h"
-#include "tvariant.h"
-#include "tep.h"
-#include "stub.h"
-#include "addr_any.h"
-#include "scalar.h"
-
-namespace {
-
-}
-
-TEST(scalarTest, func) {
-
-}
+#ifdef __cplusplus
+extern "C" {
+#endif
 
+char *taosCharsetReplace(char *charsetstr);
+void  taosGetSystemLocale(char *outLocale, char *outCharset);
+void  taosSetSystemLocale(const char *inLocale, const char *inCharSet);
 
-int main(int argc, char** argv) {
-  srand(time(NULL));
-  testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
+#ifdef __cplusplus
 }
+#endif
 
-#pragma GCC diagnostic pop
+#endif /*_TD_OS_LOCALE_H_*/

include/os/osSocket.h

@@ -55,7 +55,7 @@ void    taosSetMaskSIGPIPE();
 int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen);
 int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen);
 
-uint32_t    taosInetAddr(char *ipAddr);
+uint32_t    taosInetAddr(const char *ipAddr);
 const char *taosInetNtoa(struct in_addr ipInt);
 
 #if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) 

include/os/osString.h

@@ -45,7 +45,6 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs);
 bool    taosMbsToUcs4(const char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len);
 int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize);
 bool    taosValidateEncodec(const char *encodec);
-char *  taosCharsetReplace(char *charsetstr);
 
 #ifdef __cplusplus
 }

include/os/osSysinfo.h

@@ -22,23 +22,28 @@ extern "C" {
 
 #include "os.h"
 
-#define TSDB_LOCALE_LEN   64
-#define TSDB_TIMEZONE_LEN 96
+#define TD_LOCALE_LEN   64
+#define TD_CHARSET_LEN  64
+#define TD_TIMEZONE_LEN 96
+
+typedef struct {
+  int64_t total;
+  int64_t used;
+  int64_t avail;
+} SDiskSize;
+
+typedef struct SDiskSpace {
+  int64_t   reserved;
+  SDiskSize size;
+} SDiskSpace;
 
 extern int64_t tsPageSize;
 extern int64_t tsOpenMax;
 extern int64_t tsStreamMax;
 extern int32_t tsNumOfCores;
 extern int32_t tsTotalMemoryMB;
-extern char     tsTimezone[];
-extern char     tsLocale[];
-extern char     tsCharset[];            // default encode string
 
-typedef struct {
-  int64_t total;
-  int64_t used;
-  int64_t avail;
-} SDiskSize;
+
 
 int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize);
 int32_t taosGetCpuCores();

include/os/osTimezone.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_OS_TIMEZONE_H_
+#define _TD_OS_TIMEZONE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void taosGetSystemTimezone(char *outTimezone);
+void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_OS_TIMEZONE_H_*/

include/util/taoserror.h

@@ -20,6 +20,8 @@
 extern "C" {
 #endif
 
+// clang-format off
+
 #define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code))))
  
 #define TAOS_SYSTEM_ERROR(code)             (0x80ff0000 | (code))
@@ -71,6 +73,8 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MSG_NOT_PROCESSED             TAOS_DEF_ERROR_CODE(0, 0x0109)
 #define TSDB_CODE_INVALID_PARA                  TAOS_DEF_ERROR_CODE(0, 0x010A)
 #define TSDB_CODE_REPEAT_INIT                   TAOS_DEF_ERROR_CODE(0, 0x010B)
+#define TSDB_CODE_CFG_NOT_FOUND                 TAOS_DEF_ERROR_CODE(0, 0x010C)
+#define TSDB_CODE_INVALID_CFG                   TAOS_DEF_ERROR_CODE(0, 0x010D)
 
 #define TSDB_CODE_REF_NO_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0110)
 #define TSDB_CODE_REF_FULL                      TAOS_DEF_ERROR_CODE(0, 0x0111)
@@ -260,6 +264,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_CONSUMER_NOT_EXIST        TAOS_DEF_ERROR_CODE(0, 0x03E7)
 #define TSDB_CODE_MND_UNSUPPORTED_TOPIC         TAOS_DEF_ERROR_CODE(0, 0x03E8)
 #define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E9)
+#define TSDB_CODE_MND_OFFSET_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03EA)
 #define TSDB_CODE_MND_MQ_PLACEHOLDER            TAOS_DEF_ERROR_CODE(0, 0x03F0)
 
 // dnode

include/util/tconfig.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_UTIL_CONFIG_H
-#define _TD_UTIL_CONFIG_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define TSDB_CFG_MAX_NUM    115
-#define TSDB_CFG_PRINT_LEN  23
-#define TSDB_CFG_OPTION_LEN 24
-#define TSDB_CFG_VALUE_LEN  41
-
-#define TSDB_CFG_CTYPE_B_CONFIG    1U   // can be configured from file
-#define TSDB_CFG_CTYPE_B_SHOW      2U   // can displayed by "show configs" commands
-#define TSDB_CFG_CTYPE_B_LOG       4U   // is a log type configuration
-#define TSDB_CFG_CTYPE_B_CLIENT    8U   // can be displayed in the client log
-#define TSDB_CFG_CTYPE_B_OPTION    16U  // can be configured by taos_options function
-#define TSDB_CFG_CTYPE_B_NOT_PRINT 32U  // such as password
-
-#define MAX_FLOAT    100000
-#define MIN_FLOAT    0
-
-enum {
-  TAOS_CFG_CSTATUS_NONE,     // not configured
-  TAOS_CFG_CSTATUS_DEFAULT,  // use system default value
-  TAOS_CFG_CSTATUS_FILE,     // configured from file
-  TAOS_CFG_CSTATUS_OPTION,   // configured by taos_options function
-  TAOS_CFG_CSTATUS_ARG,      // configured by program argument
-};
-
-enum {
-  TAOS_CFG_VTYPE_INT8,
-  TAOS_CFG_VTYPE_INT16,
-  TAOS_CFG_VTYPE_INT32,
-  TAOS_CFG_VTYPE_UINT16,
-  TAOS_CFG_VTYPE_FLOAT,
-  TAOS_CFG_VTYPE_STRING,
-  TAOS_CFG_VTYPE_IPSTR,
-  TAOS_CFG_VTYPE_DIRECTORY,
-  TAOS_CFG_VTYPE_DATA_DIRCTORY,
-  TAOS_CFG_VTYPE_DOUBLE,
-};
-
-enum {
-  TAOS_CFG_UTYPE_NONE,
-  TAOS_CFG_UTYPE_PERCENT,
-  TAOS_CFG_UTYPE_GB,
-  TAOS_CFG_UTYPE_MB,
-  TAOS_CFG_UTYPE_BYTE,
-  TAOS_CFG_UTYPE_SECOND,
-  TAOS_CFG_UTYPE_MS
-};
-
-typedef struct {
-  char *   option;
-  void *   ptr;
-  float    minValue;
-  float    maxValue;
-  int8_t   cfgType;
-  int8_t   cfgStatus;
-  int8_t   unitType;
-  int8_t   valType;
-  int32_t  ptrLength;
-} SGlobalCfg;
-
-extern SGlobalCfg tsGlobalConfig[];
-extern int32_t    tsGlobalConfigNum;
-extern char *     tsCfgStatusStr[];
-
-void    taosReadGlobalLogCfg();
-int32_t taosReadCfgFromFile();
-void    taosPrintCfg();
-void    taosDumpGlobalCfg();
-
-void        taosAddConfigOption(SGlobalCfg cfg);
-SGlobalCfg *taosGetConfigOption(const char *option);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif  /*_TD_UTIL_CONFIG_H*/

include/util/tdef.h

@@ -13,6 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+// clang-format off
+
 #ifndef _TD_UTIL_DEF_H
 #define _TD_UTIL_DEF_H
 
@@ -108,35 +110,52 @@ do { \
   (src) = (void *)((char *)src + sizeof(type));\
 } while(0)
 
+typedef enum EOperatorType {
+  // arithmetic operator
+  OP_TYPE_ADD = 1,
+  OP_TYPE_SUB,
+  OP_TYPE_MULTI,
+  OP_TYPE_DIV,
+  OP_TYPE_MOD,
+
+  // bit operator
+  OP_TYPE_BIT_AND,
+  OP_TYPE_BIT_OR,
+
+  // comparison operator
+  OP_TYPE_GREATER_THAN,
+  OP_TYPE_GREATER_EQUAL,
+  OP_TYPE_LOWER_THAN,
+  OP_TYPE_LOWER_EQUAL,
+  OP_TYPE_EQUAL,
+  OP_TYPE_NOT_EQUAL,
+  OP_TYPE_IN,
+  OP_TYPE_NOT_IN,
+  OP_TYPE_LIKE,
+  OP_TYPE_NOT_LIKE,
+  OP_TYPE_MATCH,
+  OP_TYPE_NMATCH,
+  OP_TYPE_IS_NULL,
+  OP_TYPE_IS_NOT_NULL,
+  OP_TYPE_IS_TRUE,
+  OP_TYPE_IS_FALSE,
+  OP_TYPE_IS_UNKNOWN,
+  OP_TYPE_IS_NOT_TRUE,
+  OP_TYPE_IS_NOT_FALSE,
+  OP_TYPE_IS_NOT_UNKNOWN,
+
+  // json operator
+  OP_TYPE_JSON_GET_VALUE,
+  OP_TYPE_JSON_CONTAINS
+} EOperatorType;
+
+
+typedef enum ELogicConditionType {
+  LOGIC_COND_TYPE_AND,
+  LOGIC_COND_TYPE_OR,
+  LOGIC_COND_TYPE_NOT,
+} ELogicConditionType;
 
-// TODO: check if below is necessary
-#define TSDB_RELATION_INVALID     0
-#define TSDB_RELATION_LESS        1
-#define TSDB_RELATION_GREATER     2
-#define TSDB_RELATION_EQUAL       3
-#define TSDB_RELATION_LESS_EQUAL  4
-#define TSDB_RELATION_GREATER_EQUAL 5
-#define TSDB_RELATION_NOT_EQUAL   6
-#define TSDB_RELATION_LIKE        7
-#define TSDB_RELATION_NOT_LIKE    8
-#define TSDB_RELATION_ISNULL      9
-#define TSDB_RELATION_NOTNULL     10
-#define TSDB_RELATION_IN          11
-#define TSDB_RELATION_NOT_IN      12
-
-#define TSDB_RELATION_AND         13
-#define TSDB_RELATION_OR          14
-#define TSDB_RELATION_NOT         15
-
-#define TSDB_RELATION_MATCH       16
-#define TSDB_RELATION_NMATCH      17
-
-#define TSDB_BINARY_OP_ADD        4000
-#define TSDB_BINARY_OP_SUBTRACT   4001
-#define TSDB_BINARY_OP_MULTIPLY   4002
-#define TSDB_BINARY_OP_DIVIDE     4003
-#define TSDB_BINARY_OP_REMAINDER  4004
-#define TSDB_BINARY_OP_CONCAT     4005
 
 #define FUNCTION_CEIL        4500
 #define FUNCTION_FLOOR       4501
@@ -148,9 +167,6 @@ do { \
 #define FUNCTION_LTRIM       4802
 #define FUNCTION_RTRIM       4803
 
-#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN))
-#define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER))
-
 #define TSDB_NAME_DELIMITER_LEN   1
 
 #define TSDB_UNI_LEN              24
@@ -182,6 +198,7 @@ do { \
 #define TSDB_TOPIC_FNAME_LEN      TSDB_TABLE_FNAME_LEN
 #define TSDB_CONSUMER_GROUP_LEN   192
 #define TSDB_SUBSCRIBE_KEY_LEN    (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2)
+#define TSDB_PARTITION_KEY_LEN    (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2)
 #define TSDB_COL_NAME_LEN         65
 #define TSDB_MAX_SAVED_SQL_LEN    TSDB_MAX_COLUMNS * 64
 #define TSDB_MAX_SQL_LEN          TSDB_PAYLOAD_SIZE
@@ -335,9 +352,6 @@ do { \
 #define TSDB_QUERY_TYPE_NON_TYPE        0x00u     // none type
 #define TSDB_QUERY_TYPE_FREE_RESOURCE   0x01u     // free qhandle at vnode
 
-#define TSDB_QUERY_TYPE_NON_TYPE      0x00u  // none type
-#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u  // free qhandle at vnode
-
 #define TSDB_META_COMPACT_RATIO         0       // disable tsdb meta compact by default
 
 

include/util/tlog.h

@@ -22,8 +22,8 @@
 extern "C" {
 #endif
 
-// log
-extern int8_t  tsAsyncLog;
+extern bool    tsLogInited;
+extern bool    tsAsyncLog;
 extern int32_t tsNumOfLogLines;
 extern int32_t tsLogKeepDays;
 extern int32_t dDebugFlag;
@@ -32,9 +32,6 @@ extern int32_t mDebugFlag;
 extern int32_t cDebugFlag;
 extern int32_t jniDebugFlag;
 extern int32_t tmrDebugFlag;
-extern int32_t httpDebugFlag;
-extern int32_t mqttDebugFlag;
-extern int32_t monDebugFlag;
 extern int32_t uDebugFlag;
 extern int32_t rpcDebugFlag;
 extern int32_t qDebugFlag;
@@ -42,8 +39,7 @@ extern int32_t wDebugFlag;
 extern int32_t sDebugFlag;
 extern int32_t tsdbDebugFlag;
 extern int32_t tqDebugFlag;
-extern int32_t cqDebugFlag;
-extern int32_t debugFlag;
+extern int32_t fsDebugFlag;
 
 #define DEBUG_FATAL  1U
 #define DEBUG_ERROR  DEBUG_FATAL
@@ -52,13 +48,14 @@ extern int32_t debugFlag;
 #define DEBUG_DEBUG  4U
 #define DEBUG_TRACE  8U
 #define DEBUG_DUMP   16U
-
 #define DEBUG_SCREEN 64U
 #define DEBUG_FILE   128U
 
-int32_t taosInitLog(char *logName, int32_t numOfLogLines, int32_t maxFiles);
+int32_t taosInitLog(const char *logName, int32_t maxFiles);
 void    taosCloseLog();
 void    taosResetLog();
+void    taosSetAllDebugFlag(int32_t flag);
+void    taosDumpData(unsigned char *msg, int32_t len);
 
 void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...)
 #ifdef __GNUC__
@@ -72,8 +69,6 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, .
 #endif
     ;
 
-void taosDumpData(unsigned char *msg, int32_t len);
-
 #ifdef __cplusplus
 }
 #endif

include/util/ttimer.h

@@ -26,7 +26,6 @@ typedef void *tmr_h;
 typedef void (*TAOS_TMR_CALLBACK)(void *, void *);
 
 extern int      taosTmrThreads;
-extern uint32_t tsMaxTmrCtrl;
 
 #define MSECONDS_PER_TICK 5
 

include/util/version.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_UTIL_VERSION_H
+#define _TD_UTIL_VERSION_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern char version[];
+extern char compatible_version[];
+extern char gitinfo[];
+extern char gitinfoOfInternal[];
+extern char buildinfo[];
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_VERSION_H*/

source/client/CMakeLists.txt

@@ -8,7 +8,7 @@ target_include_directories(
 target_link_libraries(
     taos
     INTERFACE api
-    PRIVATE os util common transport parser planner catalog scheduler function qcom
+    PRIVATE os util common transport parser planner catalog scheduler function qcom config
 )
 
 if(${BUILD_TEST})

source/client/inc/clientInt.h

@@ -20,17 +20,19 @@
 extern "C" {
 #endif
 
-#include "taos.h"
 #include "common.h"
-#include "tmsg.h"
+#include "parser.h"
+#include "query.h"
+#include "taos.h"
 #include "tdef.h"
 #include "tep.h"
 #include "thash.h"
 #include "tlist.h"
+#include "tmsg.h"
 #include "tmsgtype.h"
 #include "trpc.h"
-#include "query.h"
-#include "parser.h"
+
+#include "config.h"
 
 #define CHECK_CODE_GOTO(expr, label) \
   do {                               \
@@ -46,12 +48,12 @@ extern "C" {
 typedef struct SAppInstInfo SAppInstInfo;
 
 typedef struct SHbConnInfo {
-  void         *param;
-  SClientHbReq *req;
+  void*         param;
+  SClientHbReq* req;
 } SHbConnInfo;
 
 typedef struct SAppHbMgr {
-  char   *key;
+  char* key;
   // statistics
   int32_t reportCnt;
   int32_t connKeyCnt;
@@ -66,11 +68,9 @@ typedef struct SAppHbMgr {
   SHashObj* connInfo;    // hash<SClientHbKey, SHbConnInfo>
 } SAppHbMgr;
 
+typedef int32_t (*FHbRspHandle)(struct SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp);
 
-typedef int32_t (*FHbRspHandle)(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp);
-
-typedef int32_t (*FHbReqHandle)(SClientHbKey *connKey, void* param, SClientHbReq *req);
-
+typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req);
 
 typedef struct SClientHbMgr {
   int8_t inited;
@@ -83,7 +83,6 @@ typedef struct SClientHbMgr {
   FHbRspHandle    rspHandle[HEARTBEAT_TYPE_MAX];
 } SClientHbMgr;
 
-
 typedef struct SQueryExecMetric {
   int64_t start;   // start timestamp
   int64_t parsed;  // start to parse
@@ -105,26 +104,26 @@ typedef struct SInstanceSummary {
 } SInstanceSummary;
 
 typedef struct SHeartBeatInfo {
-  void  *pTimer;   // timer, used to send request msg to mnode
+  void* pTimer;  // timer, used to send request msg to mnode
 } SHeartBeatInfo;
 
 struct SAppInstInfo {
   int64_t           numOfConns;
   SCorEpSet         mgmtEp;
   SInstanceSummary  summary;
-  SList            *pConnList;  // STscObj linked list
+  SList*            pConnList;  // STscObj linked list
   int64_t           clusterId;
-  void             *pTransporter;
-  struct SAppHbMgr *pAppHbMgr;
+  void*             pTransporter;
+  struct SAppHbMgr* pAppHbMgr;
 };
 
 typedef struct SAppInfo {
   int64_t         startTime;
   char            appName[TSDB_APP_NAME_LEN];
-  char           *ep;
+  char*           ep;
   int32_t         pid;
   int32_t         numOfThreads;
-  SHashObj       *pInstMap;
+  SHashObj*       pInstMap;
   pthread_mutex_t mutex;
 } SAppInfo;
 
@@ -139,7 +138,7 @@ typedef struct STscObj {
   uint64_t        id;         // ref ID returned by taosAddRef
   pthread_mutex_t mutex;      // used to protect the operation on db
   int32_t         numOfReqs;  // number of sqlObj bound to this connection
-  SAppInstInfo    *pAppInfo;
+  SAppInstInfo*   pAppInfo;
 } STscObj;
 
 typedef struct SMqConsumer {
@@ -147,13 +146,13 @@ typedef struct SMqConsumer {
 } SMqConsumer;
 
 typedef struct SReqResultInfo {
-  const char  *pRspMsg;
-  const char  *pData;
-  TAOS_FIELD  *fields;
+  const char* pRspMsg;
+  const char* pData;
+  TAOS_FIELD* fields;
   uint32_t    numOfCols;
-  int32_t     *length;
+  int32_t*    length;
   TAOS_ROW    row;
-  char       **pCol;
+  char**      pCol;
   uint32_t    numOfRows;
   uint64_t    totalRows;
   uint32_t    current;
@@ -163,7 +162,7 @@ typedef struct SReqResultInfo {
 typedef struct SShowReqInfo {
   int64_t execId;  // showId/queryId
   int32_t vgId;
-  SArray         *pArray;        // SArray<SVgroupInfo>
+  SArray* pArray;        // SArray<SVgroupInfo>
   int32_t currentIndex;  // current accessed vgroup index.
 } SShowReqInfo;
 
@@ -172,8 +171,8 @@ typedef struct SRequestSendRecvBody {
   void*             fp;
   SShowReqInfo      showInfo;  // todo this attribute will be removed after the query framework being completed.
   SDataBuf          requestMsg;
-  struct SSchJob   *pQueryJob;     // query job, created according to sql query DAG.
-  struct SQueryDag *pDag;          // the query dag, generated according to the sql statement.
+  struct SSchJob*   pQueryJob;  // query job, created according to sql query DAG.
+  struct SQueryDag* pDag;       // the query dag, generated according to the sql statement.
   SReqResultInfo    resInfo;
 } SRequestSendRecvBody;
 
@@ -182,12 +181,12 @@ typedef struct SRequestSendRecvBody {
 typedef struct SRequestObj {
   uint64_t             requestId;
   int32_t              type;  // request type
-  STscObj         *pTscObj;
-  char            *sqlstr;  // sql string
+  STscObj*             pTscObj;
+  char*                sqlstr;  // sql string
   int32_t              sqlLen;
   int64_t              self;
-  char            *msgBuf;
-  void            *pInfo;   // sql parse info, generated by parser module
+  char*                msgBuf;
+  void*                pInfo;  // sql parse info, generated by parser module
   int32_t              code;
   SQueryExecMetric     metric;
   SRequestSendRecvBody body;
@@ -203,34 +202,35 @@ SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj);
 
 int taos_init();
 
-void* createTscObj(const char* user, const char* auth, const char *db, SAppInstInfo* pAppInfo);
-void  destroyTscObj(void*pObj);
+void* createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo);
+void  destroyTscObj(void* pObj);
 
 uint64_t generateRequestId();
 
-void *createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type);
+void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type);
 void  destroyRequest(SRequestObj* pRequest);
 
-char *getDbOfConnection(STscObj* pObj);
+char* getDbOfConnection(STscObj* pObj);
 void  setConnectionDB(STscObj* pTscObj, const char* db);
 
 void taos_init_imp(void);
-int  taos_options_imp(TSDB_OPTION option, const char *str);
+int  taos_options_imp(TSDB_OPTION option, const char* str);
 
-void* openTransporter(const char *user, const char *auth, int32_t numOfThreads);
+void* openTransporter(const char* user, const char* auth, int32_t numOfThreads);
 
+bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType);
 void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet);
 
 void initMsgHandleFp();
 
-TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port);
+TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db,
+                            uint16_t port);
 
-void *doFetchRow(SRequestObj* pRequest);
+void* doFetchRow(SRequestObj* pRequest);
 
 void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows);
 
-
-int32_t buildRequest(STscObj *pTscObj, const char *sql, int sqlLen, SRequestObj** pRequest);
+int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest);
 
 int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery);
 
@@ -241,7 +241,7 @@ void hbMgrCleanUp();
 int  hbHandleRsp(SClientHbBatchRsp* hbRsp);
 
 // cluster level
-SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key);
+SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key);
 void       appHbMgrCleanup(void);
 
 // conn level
@@ -254,6 +254,12 @@ int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* v
 void hbMgrInitMqHbRspHandle();
 
 
+// config
+int32_t tscInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl);
+int32_t tscInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl);
+
+extern SConfig *tscCfg;
+
 #ifdef __cplusplus
 }
 #endif

source/client/src/clientCfg.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "clientInt.h"
+#include "ulog.h"
+
+// todo refact
+SConfig *tscCfg;
+
+static int32_t tscLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl) {
+  char cfgDir[PATH_MAX] = {0};
+  char cfgFile[PATH_MAX + 100] = {0};
+
+  taosExpandDir(inputCfgDir, cfgDir, PATH_MAX);
+  snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir);
+
+  if (cfgLoad(pConfig, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) {
+    uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr());
+    return -1;
+  }
+
+  if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, cfgFile) != 0) {
+    if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, cfgDir) != 0) {
+      uError("failed to load from config file:%s since %s\n", cfgFile, terrstr());
+      return -1;
+    }
+  }
+
+  if (cfgLoad(pConfig, CFG_STYPE_ENV_FILE, envFile) != 0) {
+    uError("failed to load from env file:%s since %s\n", envFile, terrstr());
+    return -1;
+  }
+
+  if (cfgLoad(pConfig, CFG_STYPE_ENV_VAR, NULL) != 0) {
+    uError("failed to load from global env variables since %s\n", terrstr());
+    return -1;
+  }
+
+  return 0;
+}
+
+static int32_t tscAddLogCfg(SConfig *pCfg) {
+  if (cfgAddDir(pCfg, "logDir", "/var/log/taos") != 0) return -1;
+  if (cfgAddBool(pCfg, "asyncLog", 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "numOfLogLines", 10000000, 1000, 2000000000) != 0) return -1;
+  if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000) != 0) return -1;
+  if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "cDebugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "jniDebugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "tmrDebugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "uDebugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "rpcDebugFlag", 0, 0, 255) != 0) return -1;
+  return 0;
+}
+
+static int32_t tscSetLogCfg(SConfig *pCfg) {
+  osSetLogDir(cfgGetItem(pCfg, "logDir")->str);
+  tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval;
+  tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32;
+  tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32;
+  cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32;
+  jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32;
+  tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32;
+  uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32;
+  rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32;
+
+  int32_t debugFlag = cfgGetItem(pCfg, "debugFlag")->i32;
+  taosSetAllDebugFlag(debugFlag);
+  return 0;
+}
+
+int32_t tscInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl) {
+  if (tsLogInited) return 0;
+
+  SConfig *pCfg = cfgInit();
+  if (pCfg == NULL) return -1;
+
+  if (tscAddLogCfg(pCfg) != 0) {
+    printf("failed to add log cfg since %s\n", terrstr());
+    cfgCleanup(pCfg);
+    return -1;
+  }
+
+  if (tscLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) {
+    printf("failed to load log cfg since %s\n", terrstr());
+    cfgCleanup(pCfg);
+    return -1;
+  }
+
+  if (tscSetLogCfg(pCfg) != 0) {
+    printf("failed to set log cfg since %s\n", terrstr());
+    cfgCleanup(pCfg);
+    return -1;
+  }
+
+  const int32_t maxLogFileNum = 10;
+  if (taosInitLog("taoslog", maxLogFileNum) != 0) {
+    printf("failed to init log file since %s\n", terrstr());
+    cfgCleanup(pCfg);
+    return -1;
+  }
+
+  cfgDumpCfg(pCfg);
+  cfgCleanup(pCfg);
+  return 0;
+}
+
+static int32_t tscAddEpCfg(SConfig *pCfg) {
+  char defaultFqdn[TSDB_FQDN_LEN] = {0};
+  if (taosGetFqdn(defaultFqdn) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+  if (cfgAddString(pCfg, "fqdn", defaultFqdn) != 0) return -1;
+
+  int32_t defaultServerPort = 6030;
+  if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056) != 0) return -1;
+
+  char defaultFirstEp[TSDB_EP_LEN] = {0};
+  char defaultSecondEp[TSDB_EP_LEN] = {0};
+  snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort);
+  snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort);
+  if (cfgAddString(pCfg, "firstEp", defaultFirstEp) != 0) return -1;
+  if (cfgAddString(pCfg, "secondEp", defaultSecondEp) != 0) return -1;
+
+  return 0;
+}
+
+static int32_t tscAddCfg(SConfig *pCfg) {
+  if (tscAddEpCfg(pCfg) != 0) return -1;
+
+  // if (cfgAddString(pCfg, "buildinfo", buildinfo) != 0) return -1;
+  // if (cfgAddString(pCfg, "gitinfo", gitinfo) != 0) return -1;
+  // if (cfgAddString(pCfg, "version", version) != 0) return -1;
+
+  // if (cfgAddDir(pCfg, "dataDir", osDataDir()) != 0) return -1;
+  if (cfgAddTimezone(pCfg, "timezone", "") != 0) return -1;
+  if (cfgAddLocale(pCfg, "locale", "") != 0) return -1;
+  if (cfgAddCharset(pCfg, "charset", "") != 0) return -1;
+  if (cfgAddInt32(pCfg, "numOfCores", 1, 1, 100000) != 0) return -1;
+  if (cfgAddInt32(pCfg, "numOfCommitThreads", 4, 1, 1000) != 0) return -1;
+  // if (cfgAddBool(pCfg, "telemetryReporting", 0) != 0) return -1;
+  if (cfgAddBool(pCfg, "enableCoreFile", 0) != 0) return -1;
+  // if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536) != 0) return -1;
+  if (cfgAddInt32(pCfg, "statusInterval", 1, 1, 30) != 0) return -1;
+  if (cfgAddFloat(pCfg, "numOfThreadsPerCore", 1, 0, 10) != 0) return -1;
+  if (cfgAddFloat(pCfg, "ratioOfQueryCores", 1, 0, 5) != 0) return -1;
+  if (cfgAddInt32(pCfg, "shellActivityTimer", 3, 1, 120) != 0) return -1;
+  if (cfgAddInt32(pCfg, "rpcTimer", 300, 100, 3000) != 0) return -1;
+  if (cfgAddInt32(pCfg, "rpcMaxTime", 600, 100, 7200) != 0) return -1;
+  if (cfgAddInt32(pCfg, "maxConnections", 50000, 1, 100000) != 0) return -1;
+  return 0;
+}
+
+int32_t tscCheckCfg(SConfig *pCfg) {
+  bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval;
+  taosSetCoreDump(enableCore);
+
+  return 0;
+}
+
+SConfig *tscInitCfgImp(const char *cfgDir, const char *envFile, const char *apolloUrl) {
+  SConfig *pCfg = cfgInit();
+  if (pCfg == NULL) return NULL;
+
+  if (tscAddCfg(pCfg) != 0) {
+    uError("failed to init tsc cfg since %s", terrstr());
+    cfgCleanup(pCfg);
+    return NULL;
+  }
+
+  if (tscLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) {
+    printf("failed to load tsc cfg since %s\n", terrstr());
+    cfgCleanup(pCfg);
+    return NULL;
+  }
+
+  if (tscCheckCfg(pCfg) != 0) {
+    uError("failed to check cfg since %s", terrstr());
+    cfgCleanup(pCfg);
+    return NULL;
+  }
+
+  cfgDumpCfg(pCfg);
+  return pCfg;
+}
+
+int32_t tscInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl) {
+  tscCfg = tscInitCfgImp(cfgDir, envFile, apolloUrl);
+  if (tscCfg == NULL) return -1;
+
+  return 0;
+}
\ No newline at end of file

source/client/src/clientEnv.c

@@ -13,21 +13,18 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "os.h"
 #include "catalog.h"
 #include "clientInt.h"
 #include "clientLog.h"
+#include "os.h"
 #include "query.h"
 #include "scheduler.h"
-#include "tmsg.h"
 #include "tcache.h"
-#include "tconfig.h"
 #include "tglobal.h"
-#include "tnote.h"
+#include "tmsg.h"
 #include "tref.h"
 #include "trpc.h"
 #include "ttime.h"
-#include "ttimezone.h"
 
 #define TSC_VAR_NOT_RELEASE 1
 #define TSC_VAR_RELEASED 0
@@ -39,7 +36,7 @@ int32_t    clientConnRefPool = -1;
 static pthread_once_t tscinit = PTHREAD_ONCE_INIT;
 volatile int32_t      tscInitRes = 0;
 
-static void registerRequest(SRequestObj* pRequest) {
+static void registerRequest(SRequestObj *pRequest) {
   STscObj *pTscObj = (STscObj *)taosAcquireRef(clientConnRefPool, pRequest->pTscObj->id);
   assert(pTscObj != NULL);
 
@@ -53,69 +50,58 @@ static void registerRequest(SRequestObj* pRequest) {
 
     int32_t total = atomic_add_fetch_32(&pSummary->totalRequests, 1);
     int32_t currentInst = atomic_add_fetch_32(&pSummary->currentRequests, 1);
-    tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 ", current:%d, app current:%d, total:%d, reqId:0x%"PRIx64, pRequest->self,
-             pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
+    tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64
+             ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64,
+             pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
   }
 }
 
-static void deregisterRequest(SRequestObj* pRequest) {
+static void deregisterRequest(SRequestObj *pRequest) {
   assert(pRequest != NULL);
 
-  STscObj* pTscObj = pRequest->pTscObj;
-  SInstanceSummary* pActivity = &pTscObj->pAppInfo->summary;
+  STscObj *         pTscObj = pRequest->pTscObj;
+  SInstanceSummary *pActivity = &pTscObj->pAppInfo->summary;
 
   int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1);
   int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1);
 
   int64_t duration = taosGetTimestampMs() - pRequest->metric.start;
-  tscDebug("0x%"PRIx64" free Request from connObj: 0x%"PRIx64", reqId:0x%"PRIx64" elapsed:%"PRIu64" ms, current:%d, app current:%d", pRequest->self, pTscObj->id,
-      pRequest->requestId, duration, num, currentInst);
+  tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64
+           " ms, current:%d, app current:%d",
+           pRequest->self, pTscObj->id, pRequest->requestId, duration, num, currentInst);
   taosReleaseRef(clientConnRefPool, pTscObj->id);
 }
 
-static void tscInitLogFile() {
-  taosReadGlobalLogCfg();
-  if (mkdir(tsLogDir, 0755) != 0 && errno != EEXIST) {
-    printf("failed to create log dir:%s\n", tsLogDir);
-  }
-
-  const char    *defaultLogFileNamePrefix = "taoslog";
-  const int32_t  maxLogFileNum = 10;
 
-  char temp[128] = {0};
-  sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix);
-  if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) {
-    printf("failed to open log file in directory:%s\n", tsLogDir);
-  }
-}
 
 // todo close the transporter properly
-void closeTransporter(STscObj* pTscObj)  {
+void closeTransporter(STscObj *pTscObj) {
   if (pTscObj == NULL || pTscObj->pAppInfo->pTransporter == NULL) {
     return;
   }
 
-  tscDebug("free transporter:%p in connObj: 0x%"PRIx64, pTscObj->pAppInfo->pTransporter, pTscObj->id);
+  tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, pTscObj->id);
   rpcClose(pTscObj->pAppInfo->pTransporter);
 }
 
 // TODO refactor
-void* openTransporter(const char *user, const char *auth, int32_t numOfThread) {
+void *openTransporter(const char *user, const char *auth, int32_t numOfThread) {
   SRpcInit rpcInit;
   memset(&rpcInit, 0, sizeof(rpcInit));
   rpcInit.localPort = 0;
   rpcInit.label = "TSC";
   rpcInit.numOfThreads = numOfThread;
   rpcInit.cfp = processMsgFromServer;
-  rpcInit.sessions = tsMaxConnections;
+  rpcInit.pfp = persistConnForSpecificMsg;
+  rpcInit.sessions = cfgGetItem(tscCfg, "maxConnections")->i32;
   rpcInit.connType = TAOS_CONN_CLIENT;
   rpcInit.user = (char *)user;
-  rpcInit.idleTime = tsShellActivityTimer * 1000;
+  rpcInit.idleTime = cfgGetItem(tscCfg, "shellActivityTimer")->i32 * 1000;
   rpcInit.ckey = "key";
   rpcInit.spi = 1;
   rpcInit.secret = (char *)auth;
 
-  void* pDnodeConn = rpcOpen(&rpcInit);
+  void *pDnodeConn = rpcOpen(&rpcInit);
   if (pDnodeConn == NULL) {
     tscError("failed to init connection to server");
     return NULL;
@@ -130,12 +116,12 @@ void destroyTscObj(void *pObj) {
   SClientHbKey connKey = {.connId = pTscObj->connId, .hbType = pTscObj->connType};
   hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey);
   atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
-  tscDebug("connObj 0x%"PRIx64" destroyed, totalConn:%"PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns);
+  tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns);
   pthread_mutex_destroy(&pTscObj->mutex);
   tfree(pTscObj);
 }
 
-void* createTscObj(const char* user, const char* auth, const char *db, SAppInstInfo* pAppInfo) {
+void *createTscObj(const char *user, const char *auth, const char *db, SAppInstInfo *pAppInfo) {
   STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj));
   if (NULL == pObj) {
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
@@ -153,11 +139,11 @@ void* createTscObj(const char* user, const char* auth, const char *db, SAppInstI
   pthread_mutex_init(&pObj->mutex, NULL);
   pObj->id = taosAddRef(clientConnRefPool, pObj);
 
-  tscDebug("connObj created, 0x%"PRIx64, pObj->id);
+  tscDebug("connObj created, 0x%" PRIx64, pObj->id);
   return pObj;
 }
 
-void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type) {
+void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t type) {
   assert(pObj != NULL);
 
   SRequestObj *pRequest = (SRequestObj *)calloc(1, sizeof(SRequestObj));
@@ -179,7 +165,7 @@ void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t ty
   return pRequest;
 }
 
-static void doFreeReqResultInfo(SReqResultInfo* pResInfo) {
+static void doFreeReqResultInfo(SReqResultInfo *pResInfo) {
   tfree(pResInfo->pRspMsg);
   tfree(pResInfo->length);
   tfree(pResInfo->row);
@@ -187,9 +173,9 @@ static void doFreeReqResultInfo(SReqResultInfo* pResInfo) {
   tfree(pResInfo->fields);
 }
 
-static void doDestroyRequest(void* p) {
+static void doDestroyRequest(void *p) {
   assert(p != NULL);
-  SRequestObj* pRequest = (SRequestObj*)p;
+  SRequestObj *pRequest = (SRequestObj *)p;
 
   assert(RID_VALID(pRequest->self));
 
@@ -208,7 +194,7 @@ static void doDestroyRequest(void* p) {
   tfree(pRequest);
 }
 
-void destroyRequest(SRequestObj* pRequest) {
+void destroyRequest(SRequestObj *pRequest) {
   if (pRequest == NULL) {
     return;
   }
@@ -225,27 +211,32 @@ void taos_init_imp(void) {
   srand(taosGetTimestampSec());
 
   deltaToUtcInitOnce();
-  taosInitGlobalCfg();
-  taosReadCfgFromFile();
 
-  tscInitLogFile();
-  if (taosCheckAndPrintCfg()) {
+  if (tscInitLog(configDir, NULL, NULL) != 0) {
+    tscInitRes = -1;
+    return;
+  }
+
+  if (tscInitCfg(configDir, NULL, NULL) != 0) {
     tscInitRes = -1;
     return;
   }
 
-  taosInitNotes();
   initMsgHandleFp();
   initQueryModuleMsgHandle();
 
-  rpcInit();
+  SRpcCfg rpcCfg = {0};
+  rpcCfg.rpcTimer = cfgGetItem(tscCfg, "rpcTimer")->i32;
+  rpcCfg.rpcMaxTime = cfgGetItem(tscCfg, "rpcMaxTime")->i32;
+  rpcCfg.sver = 30000000;
+  rpcInit(&rpcCfg);
 
   SCatalogCfg cfg = {.maxDBCacheNum = 100, .maxTblCacheNum = 100};
   catalogInit(&cfg);
 
   SSchedulerCfg scfg = {.maxJobNum = 100};
   schedulerInit(&scfg);
-  tscDebug("starting to initialize TAOS driver, local ep: %s", tsLocalEp);
+  tscDebug("starting to initialize TAOS driver");
 
   taosSetCoreDump(true);
 
@@ -269,6 +260,7 @@ int taos_init() {
 }
 
 int taos_options_imp(TSDB_OPTION option, const char *str) {
+#if 0  
   SGlobalCfg *cfg = NULL;
 
   switch (option) {
@@ -281,7 +273,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
         cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
         tscInfo("set config file directory:%s", str);
       } else {
-        tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
+        tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str,
+                tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
       }
       break;
 
@@ -296,7 +289,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
         cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
         tscInfo("set shellActivityTimer:%d", tsShellActivityTimer);
       } else {
-        tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr);
+        tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str,
+                tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr);
       }
       break;
 
@@ -305,7 +299,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
       assert(cfg != NULL);
 
       size_t len = strlen(str);
-      if (len == 0 || len > TSDB_LOCALE_LEN) {
+      if (len == 0 || len > TD_LOCALE_LEN) {
         tscInfo("Invalid locale:%s, use default", str);
         return -1;
       }
@@ -314,7 +308,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
         char sep = '.';
 
         if (strlen(tsLocale) == 0) {  // locale does not set yet
-          char* defaultLocale = setlocale(LC_CTYPE, "");
+          char *defaultLocale = setlocale(LC_CTYPE, "");
 
           // The locale of the current OS does not be set correctly, so the default locale cannot be acquired.
           // The launch of current system will abort soon.
@@ -323,7 +317,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
             return -1;
           }
 
-          tstrncpy(tsLocale, defaultLocale, TSDB_LOCALE_LEN);
+          tstrncpy(tsLocale, defaultLocale, TD_LOCALE_LEN);
         }
 
         // set the user specified locale
@@ -337,7 +331,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
           tscInfo("failed to set locale:%s, current locale:%s", str, tsLocale);
         }
 
-        tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN);
+        tstrncpy(tsLocale, locale, TD_LOCALE_LEN);
 
         char *charset = strrchr(tsLocale, sep);
         if (charset != NULL) {
@@ -352,7 +346,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
               tscInfo("charset changed from %s to %s", tsCharset, charset);
             }
 
-            tstrncpy(tsCharset, charset, TSDB_LOCALE_LEN);
+            tstrncpy(tsCharset, charset, TD_LOCALE_LEN);
             cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
 
           } else {
@@ -364,7 +358,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
           tscInfo("charset remains:%s", tsCharset);
         }
       } else {
-        tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
+        tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str,
+                tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
       }
       break;
     }
@@ -375,7 +370,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
       assert(cfg != NULL);
 
       size_t len = strlen(str);
-      if (len == 0 || len > TSDB_LOCALE_LEN) {
+      if (len == 0 || len > TD_LOCALE_LEN) {
         tscInfo("failed to set charset:%s", str);
         return -1;
       }
@@ -388,13 +383,14 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
             tscInfo("charset changed from %s to %s", tsCharset, str);
           }
 
-          tstrncpy(tsCharset, str, TSDB_LOCALE_LEN);
+          tstrncpy(tsCharset, str, TD_LOCALE_LEN);
           cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
         } else {
           tscInfo("charset:%s not valid", str);
         }
       } else {
-        tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
+        tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str,
+                tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
       }
 
       break;
@@ -405,12 +401,13 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
       assert(cfg != NULL);
 
       if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
-        tstrncpy(tsTimezone, str, TSDB_TIMEZONE_LEN);
+        tstrncpy(tsTimezone, str, TD_TIMEZONE_LEN);
         tsSetTimeZone();
         cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
         tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, str);
       } else {
-        tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
+        tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str,
+                tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr);
       }
       break;
 
@@ -419,7 +416,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
       tscError("Invalid option %d", option);
       return -1;
   }
-
+#endif
   return 0;
 }
 

source/client/src/clientImpl.c

@@ -8,11 +8,10 @@
 #include "tep.h"
 #include "tglobal.h"
 #include "tmsgtype.h"
-#include "tnote.h"
 #include "tpagedbuf.h"
 #include "tref.h"
 
-static int32_t       initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet);
+static int32_t       initEpSetFromCfg(const char* ip, uint16_t port, SCorEpSet* pEpSet);
 static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest);
 static void          destroySendMsgInfo(SMsgSendInfo* pMsgBody);
 static void          setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp);
@@ -81,19 +80,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass,
   }
 
   SCorEpSet epSet = {0};
-  if (ip) {
-    if (initEpSetFromCfg(ip, NULL, &epSet) < 0) {
-      return NULL;
-    }
-
-    if (port) {
-      epSet.epSet.eps[0].port = port;
-    }
-  } else {
-    if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) {
-      return NULL;
-    }
-  }
+  initEpSetFromCfg(ip, port, &epSet);
 
   char*          key = getClusterKey(user, secretEncrypt, ip, port);
   SAppInstInfo** pInst = NULL;
@@ -254,8 +241,6 @@ TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) {
     return NULL;
   }
 
-  nPrintTsc("%s", sql)
-
   SRequestObj* pRequest = NULL;
   SQueryNode*  pQueryNode = NULL;
   SArray*      pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
@@ -282,33 +267,41 @@ _return:
   return pRequest;
 }
 
-int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet) {
-  pEpSet->version = 0;
+int initEpSetFromCfg(const char* ip, uint16_t port, SCorEpSet* pEpSet) {
+  SConfigItem* pFirst = cfgGetItem(tscCfg, "firstEp");
+  SConfigItem* pSecond = cfgGetItem(tscCfg, "secondEp");
+  SConfigItem* pPort = cfgGetItem(tscCfg, "serverPort");
 
   // init mnode ip set
   SEpSet* mgmtEpSet = &(pEpSet->epSet);
   mgmtEpSet->numOfEps = 0;
   mgmtEpSet->inUse = 0;
+  pEpSet->version = 0;
 
-  if (firstEp && firstEp[0] != 0) {
-    if (strlen(firstEp) >= TSDB_EP_LEN) {
+  if (ip != NULL) {
+    taosGetFqdnPortFromEp(ip, (uint16_t)pPort->i32, &mgmtEpSet->eps[0]);
+    mgmtEpSet->numOfEps++;
+    if (port) {
+      mgmtEpSet->eps[0].port = port;
+    }
+  } else {
+    if (pFirst->str[0] != 0) {
+      if (strlen(pFirst->str) >= TSDB_EP_LEN) {
         terrno = TSDB_CODE_TSC_INVALID_FQDN;
         return -1;
       }
-
-    taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]);
+      taosGetFqdnPortFromEp(pFirst->str, (uint16_t)pPort->i32, &mgmtEpSet->eps[0]);
       mgmtEpSet->numOfEps++;
     }
-
-  if (secondEp && secondEp[0] != 0) {
-    if (strlen(secondEp) >= TSDB_EP_LEN) {
+    if (pSecond->str[0] != 0) {
+      if (strlen(pSecond->str) >= TSDB_EP_LEN) {
         terrno = TSDB_CODE_TSC_INVALID_FQDN;
         return -1;
       }
-
-    taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]);
+      taosGetFqdnPortFromEp(pSecond->str, (uint16_t)pPort->i32, &mgmtEpSet->eps[1]);
       mgmtEpSet->numOfEps++;
     }
+  }
 
   if (mgmtEpSet->numOfEps == 0) {
     terrno = TSDB_CODE_TSC_INVALID_FQDN;
@@ -370,7 +363,6 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) {
   pMsgSendInfo->fp = handleRequestRspFp[TMSG_INDEX(pMsgSendInfo->msgType)];
   pMsgSendInfo->param = pRequest;
 
-
   SConnectReq connectReq = {0};
   STscObj*    pObj = pRequest->pTscObj;
 
@@ -398,7 +390,9 @@ static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) {
   tfree(pMsgBody->msgInfo.pData);
   tfree(pMsgBody);
 }
-
+bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType) {
+  return msgType == TDMT_VND_QUERY_RSP || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP;
+}
 void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
   SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->ahandle;
   assert(pMsg->ahandle != NULL);

source/client/src/clientMain.c

@@ -7,6 +7,7 @@
 #include "tmsg.h"
 #include "tglobal.h"
 #include "catalog.h"
+#include "version.h"
 
 #define TSC_VAR_NOT_RELEASE 1
 #define TSC_VAR_RELEASED    0
@@ -56,9 +57,7 @@ void taos_cleanup(void) {
 }
 
 TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) {
-  int32_t p = (port != 0) ? port : tsServerPort;
-
-  tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db);
+  tscDebug("try to connect to %s:%u, user:%s db:%s", ip, port, user, db);
   if (user == NULL) {
     user = TSDB_DEFAULT_USER;
   }
@@ -67,7 +66,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha
     pass = TSDB_DEFAULT_PASS;
   }
 
-  return taos_connect_internal(ip, user, pass, NULL, db, p);
+  return taos_connect_internal(ip, user, pass, NULL, db, port);
 }
 
 void taos_close(TAOS* taos) {

source/client/src/tmq.c

@@ -24,7 +24,6 @@
 #include "tep.h"
 #include "tglobal.h"
 #include "tmsgtype.h"
-#include "tnote.h"
 #include "tpagedbuf.h"
 #include "tref.h"
 
@@ -47,20 +46,22 @@ struct tmq_topic_vgroup_list_t {
 struct tmq_conf_t {
   char           clientId[256];
   char           groupId[256];
-  bool auto_commit;
+  int8_t         auto_commit;
+  int8_t         resetOffset;
+  tmq_commit_cb* commit_cb;
   /*char*          ip;*/
   /*uint16_t       port;*/
-  tmq_commit_cb* commit_cb;
 };
 
 struct tmq_t {
   // conf
   char           groupId[256];
   char           clientId[256];
-  bool           autoCommit;
+  int8_t         autoCommit;
   SRWLatch       lock;
   int64_t        consumerId;
   int32_t        epoch;
+  int32_t        resetOffsetCfg;
   int64_t        status;
   tsem_t         rspSem;
   STscObj*       pTscObj;
@@ -79,7 +80,6 @@ typedef struct {
   // statistics
   int64_t pollCnt;
   // offset
-  int64_t committedOffset;
   int64_t currentOffset;
   // connection info
   int32_t vgId;
@@ -116,20 +116,16 @@ typedef struct {
 
 typedef struct {
   tmq_t* tmq;
-  SMqClientVg* pVg;
+  /*SMqClientVg* pVg;*/
   int32_t        async;
   tsem_t         rspSem;
-} SMqCommitCbParam;
-
-typedef struct {
-  tmq_t*         tmq;
-  tsem_t         rspSem;
   tmq_resp_err_t rspErr;
-} SMqResetOffsetParam;
+} SMqCommitCbParam;
 
 tmq_conf_t* tmq_conf_new() {
   tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t));
   conf->auto_commit = false;
+  conf->resetOffset = TMQ_CONF__RESET_OFFSET__LATEST;
   return conf;
 }
 
@@ -157,6 +153,20 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
       return TMQ_CONF_INVALID;
     }
   }
+  if (strcmp(key, "auto.offset.reset") == 0) {
+    if (strcmp(value, "none") == 0) {
+      conf->resetOffset = TMQ_CONF__RESET_OFFSET__NONE;
+      return TMQ_CONF_OK;
+    } else if (strcmp(value, "earliest") == 0) {
+      conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST;
+      return TMQ_CONF_OK;
+    } else if (strcmp(value, "latest") == 0) {
+      conf->resetOffset = TMQ_CONF__RESET_OFFSET__LATEST;
+      return TMQ_CONF_OK;
+    } else {
+      return TMQ_CONF_INVALID;
+    }
+  }
   return TMQ_CONF_UNKNOWN;
 }
 
@@ -190,14 +200,12 @@ int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) {
   if (pParam->tmq->commit_cb) {
     pParam->tmq->commit_cb(pParam->tmq, rspErr, NULL, NULL);
   }
-  if (!pParam->async) tsem_post(&pParam->rspSem);
-  return 0;
-}
-
-int32_t tmqResetOffsetCb(void* param, const SDataBuf* pMsg, int32_t code) {
-  SMqResetOffsetParam* pParam = (SMqResetOffsetParam*)param;
-  pParam->rspErr = code;
+  if (!pParam->async)
     tsem_post(&pParam->rspSem);
+  else {
+    tsem_destroy(&pParam->rspSem);
+    free(param);
+  }
   return 0;
 }
 
@@ -216,6 +224,7 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
   strcpy(pTmq->groupId, conf->groupId);
   pTmq->autoCommit = conf->auto_commit;
   pTmq->commit_cb = conf->commit_cb;
+  pTmq->resetOffsetCfg = conf->resetOffset;
 
   tsem_init(&pTmq->rspSem, 0, 0);
   pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1);
@@ -223,18 +232,40 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
   return pTmq;
 }
 
-tmq_resp_err_t tmq_reset_offset(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets) {
+tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) {
+  // TODO: add read write lock
   SRequestObj*   pRequest = NULL;
+  tmq_resp_err_t resp = TMQ_RESP_ERR__SUCCESS;
   // build msg
   // send to mnode
-  SMqCMResetOffsetReq req;
+  SMqCMCommitOffsetReq req;
+  SArray*              pArray = NULL;
+
+  if (offsets == NULL) {
+    pArray = taosArrayInit(0, sizeof(SMqOffset));
+    for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
+      SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
+      for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
+        SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
+        SMqOffset    offset;
+        strcpy(offset.topicName, pTopic->topicName);
+        strcpy(offset.cgroup, tmq->groupId);
+        offset.vgId = pVg->vgId;
+        offset.offset = pVg->currentOffset;
+        taosArrayPush(pArray, &offset);
+      }
+    }
+    req.num = pArray->size;
+    req.offsets = pArray->pData;
+  } else {
     req.num = offsets->cnt;
     req.offsets = (SMqOffset*)offsets->elems;
+  }
 
   SCoder encoder;
 
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER);
-  tEncodeSMqCMResetOffsetReq(&encoder, &req);
+  tEncodeSMqCMCommitOffsetReq(&encoder, &req);
   int32_t tlen = encoder.pos;
   void*   buf = malloc(tlen);
   if (buf == NULL) {
@@ -244,32 +275,41 @@ tmq_resp_err_t tmq_reset_offset(tmq_t* tmq, const tmq_topic_vgroup_list_t* offse
   tCoderClear(&encoder);
 
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, tlen, TD_ENCODER);
-  tEncodeSMqCMResetOffsetReq(&encoder, &req);
+  tEncodeSMqCMCommitOffsetReq(&encoder, &req);
   tCoderClear(&encoder);
 
-  pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_RESET_OFFSET);
+  pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_MQ_COMMIT_OFFSET);
   if (pRequest == NULL) {
     tscError("failed to malloc request");
   }
 
-  SMqResetOffsetParam param = {0};
-  tsem_init(&param.rspSem, 0, 0);
-  param.tmq = tmq;
+  SMqCommitCbParam* pParam = malloc(sizeof(SMqCommitCbParam));
+  if (pParam == NULL) {
+    return -1;
+  }
+  pParam->tmq = tmq;
+  tsem_init(&pParam->rspSem, 0, 0);
 
   pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen};
 
   SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
-  sendInfo->param = &param;
-  sendInfo->fp = tmqResetOffsetCb;
+  sendInfo->param = pParam;
+  sendInfo->fp = tmqCommitCb;
   SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
 
   int64_t transporterId = 0;
   asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
 
-  tsem_wait(&param.rspSem);
-  tsem_destroy(&param.rspSem);
+  if (!async) {
+    tsem_wait(&pParam->rspSem);
+    resp = pParam->rspErr;
+  }
 
-  return param.rspErr;
+  if (pArray) {
+    taosArrayDestroy(pArray);
+  }
+
+  return resp;
 }
 
 tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
@@ -641,8 +681,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
         // clang-format off
         SMqClientVg clientVg = {
             .pollCnt = 0,
-            .committedOffset = -1,
-            .currentOffset = -1,
+            .currentOffset = pVgEp->offset,
             .vgId = pVgEp->vgId,
             .epSet = pVgEp->epSet
         };
@@ -708,23 +747,51 @@ END:
   return 0;
 }
 
-SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, int32_t type, SMqClientTopic* pTopic,
-                                      SMqClientVg* pVg) {
+tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) {
+  const SMqOffset* pOffset = &offset->offset;
+  if (strcmp(pOffset->cgroup, tmq->groupId) != 0) {
+    return TMQ_RESP_ERR__FAIL;
+  }
+  int32_t sz = taosArrayGetSize(tmq->clientTopics);
+  for (int32_t i = 0; i < sz; i++) {
+    SMqClientTopic* clientTopic = taosArrayGet(tmq->clientTopics, i);
+    if (strcmp(clientTopic->topicName, pOffset->topicName) == 0) {
+      int32_t vgSz = taosArrayGetSize(clientTopic->vgs);
+      for (int32_t j = 0; j < vgSz; j++) {
+        SMqClientVg* pVg = taosArrayGet(clientTopic->vgs, j);
+        if (pVg->vgId == pOffset->vgId) {
+          pVg->currentOffset = pOffset->offset;
+          return TMQ_RESP_ERR__SUCCESS;
+        }
+      }
+    }
+  }
+  return TMQ_RESP_ERR__FAIL;
+}
+
+SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClientTopic* pTopic, SMqClientVg* pVg) {
+  int64_t reqOffset;
+  if (pVg->currentOffset >= 0) {
+    reqOffset = pVg->currentOffset;
+  } else {
+    if (tmq->resetOffsetCfg == TMQ_CONF__RESET_OFFSET__NONE) {
+      tscError("unable to poll since no committed offset but reset offset is set to none");
+      return NULL;
+    }
+    reqOffset = tmq->resetOffsetCfg;
+  }
+
   SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq));
   if (pReq == NULL) {
     return NULL;
   }
-  pReq->reqType = type;
+
   strcpy(pReq->topic, pTopic->topicName);
-  pReq->blockingTime = blocking_time;
-  pReq->consumerId = tmq->consumerId;
   strcpy(pReq->cgroup, tmq->groupId);
 
-  if (type == TMQ_REQ_TYPE_COMMIT_ONLY) {
-    pReq->offset = pVg->currentOffset;
-  } else {
-    pReq->offset = pVg->currentOffset + 1;
-  }
+  pReq->blockingTime = blocking_time;
+  pReq->consumerId = tmq->consumerId;
+  pReq->currentOffset = reqOffset;
 
   pReq->head.vgId = htonl(pVg->vgId);
   pReq->head.contLen = htonl(sizeof(SMqConsumeReq));
@@ -743,13 +810,13 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
 
   if (taosArrayGetSize(tmq->clientTopics) == 0) {
     tscDebug("consumer:%ld poll but not assigned", tmq->consumerId);
-    printf("over1\n");
+    /*printf("over1\n");*/
     usleep(blocking_time * 1000);
     return NULL;
   }
   SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx);
   if (taosArrayGetSize(pTopic->vgs) == 0) {
-    printf("over2\n");
+    /*printf("over2\n");*/
     usleep(blocking_time * 1000);
     return NULL;
   }
@@ -760,8 +827,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
     pTopic->nextVgIdx = (pTopic->nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs);
     SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx);
     /*printf("consume vg %d, offset %ld\n", pVg->vgId, pVg->currentOffset);*/
-    int32_t        reqType = tmq->autoCommit ? TMQ_REQ_TYPE_CONSUME_AND_COMMIT : TMQ_REQ_TYPE_CONSUME_ONLY;
-    SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, reqType, pTopic, pVg);
+    SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, pTopic, pVg);
     if (pReq == NULL) {
       ASSERT(false);
       usleep(blocking_time * 1000);
@@ -821,6 +887,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
   /*return pRequest;*/
 }
 
+#if 0
 tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) {
   if (tmq_topic_vgroup_list != NULL) {
     // TODO
@@ -831,7 +898,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v
     SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
     for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
       SMqClientVg*   pVg = taosArrayGet(pTopic->vgs, j);
-      SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, TMQ_REQ_TYPE_COMMIT_ONLY, pTopic, pVg);
+      SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, pTopic, pVg);
 
       SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME);
       pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)};
@@ -858,6 +925,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v
 
   return 0;
 }
+#endif
 
 void tmq_message_destroy(tmq_message_t* tmq_message) {
   if (tmq_message == NULL) return;

source/client/test/CMakeLists.txt

@@ -8,13 +8,13 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 ADD_EXECUTABLE(clientTest clientTests.cpp)
 TARGET_LINK_LIBRARIES(
         clientTest
-        PUBLIC os util common transport parser catalog scheduler function gtest taos qcom
+        PUBLIC os util common transport parser catalog scheduler function gtest taos qcom config
 )
 
 ADD_EXECUTABLE(tmqTest tmqTest.cpp)
 TARGET_LINK_LIBRARIES(
         tmqTest
-        PUBLIC os util common transport parser catalog scheduler function gtest taos qcom
+        PUBLIC os util common transport parser catalog scheduler function gtest taos qcom config
 )
 
 TARGET_INCLUDE_DIRECTORIES(

source/client/test/clientTests.cpp

@@ -49,7 +49,7 @@ int main(int argc, char** argv) {
 }
 
 TEST(testCase, driverInit_Test) {
-  taosInitGlobalCfg();
+  // taosInitGlobalCfg();
 //  taos_init();
 }
 

source/client/test/tmqTest.cpp

@@ -34,7 +34,7 @@ int main(int argc, char** argv) {
 }
 
 TEST(testCase, driverInit_Test) {
-  taosInitGlobalCfg();
+  // taosInitGlobalCfg();
 //  taos_init();
 }
 

source/common/src/tep.c

@@ -4,7 +4,7 @@
 #include "tglobal.h"
 #include "tlockfree.h"
 
-int taosGetFqdnPortFromEp(const char *ep, SEp* pEp) {
+int taosGetFqdnPortFromEp(const char *ep, uint16_t defaultPort, SEp* pEp) {
   pEp->port = 0;
   strcpy(pEp->fqdn, ep);
 
@@ -15,7 +15,7 @@ int taosGetFqdnPortFromEp(const char *ep, SEp* pEp) {
   }
 
   if (pEp->port == 0) {
-    pEp->port = tsServerPort;
+    pEp->port = defaultPort;
     return -1;
   }
 

source/common/src/tmsg.c

@@ -285,7 +285,7 @@ int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchR
 int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
   int32_t tlen = 0;
 
-  tlen += taosEncodeFixedU64(buf, pReq->ver);
+  tlen += taosEncodeFixedI64(buf, pReq->ver);
   tlen += taosEncodeString(buf, pReq->name);
   tlen += taosEncodeFixedU32(buf, pReq->ttl);
   tlen += taosEncodeFixedU32(buf, pReq->keep);
@@ -330,7 +330,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
 }
 
 void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
-  buf = taosDecodeFixedU64(buf, &(pReq->ver));
+  buf = taosDecodeFixedI64(buf, &(pReq->ver));
   buf = taosDecodeString(buf, &(pReq->name));
   buf = taosDecodeFixedU32(buf, &(pReq->ttl));
   buf = taosDecodeFixedU32(buf, &(pReq->keep));
@@ -380,7 +380,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
 int32_t tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) {
   int32_t tlen = 0;
 
-  tlen += taosEncodeFixedU64(buf, pReq->ver);
+  tlen += taosEncodeFixedI64(buf, pReq->ver);
   tlen += taosEncodeFixedU32(buf, taosArrayGetSize(pReq->pArray));
   for (size_t i = 0; i < taosArrayGetSize(pReq->pArray); i++) {
     SVCreateTbReq *pCreateTbReq = taosArrayGet(pReq->pArray, i);
@@ -393,7 +393,7 @@ int32_t tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) {
 void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) {
   uint32_t nsize = 0;
 
-  buf = taosDecodeFixedU64(buf, &pReq->ver);
+  buf = taosDecodeFixedI64(buf, &pReq->ver);
   buf = taosDecodeFixedU32(buf, &nsize);
   pReq->pArray = taosArrayInit(nsize, sizeof(SVCreateTbReq));
   for (size_t i = 0; i < nsize; i++) {
@@ -407,14 +407,14 @@ void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) {
 
 int32_t tSerializeSVDropTbReq(void **buf, SVDropTbReq *pReq) {
   int32_t tlen = 0;
-  tlen += taosEncodeFixedU64(buf, pReq->ver);
+  tlen += taosEncodeFixedI64(buf, pReq->ver);
   tlen += taosEncodeString(buf, pReq->name);
   tlen += taosEncodeFixedU8(buf, pReq->type);
   return tlen;
 }
 
 void *tDeserializeSVDropTbReq(void *buf, SVDropTbReq *pReq) {
-  buf = taosDecodeFixedU64(buf, &pReq->ver);
+  buf = taosDecodeFixedI64(buf, &pReq->ver);
   buf = taosDecodeString(buf, &pReq->name);
   buf = taosDecodeFixedU8(buf, &pReq->type);
   return buf;
@@ -1393,7 +1393,7 @@ int32_t tSerializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) {
 
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeCStr(&encoder, pRsp->db) < 0) return -1;
-  if (tEncodeU64(&encoder, pRsp->uid) < 0) return -1;
+  if (tEncodeI64(&encoder, pRsp->uid) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -1407,7 +1407,7 @@ int32_t tDeserializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) {
 
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pRsp->db) < 0) return -1;
-  if (tDecodeU64(&decoder, &pRsp->uid) < 0) return -1;
+  if (tDecodeI64(&decoder, &pRsp->uid) < 0) return -1;
   tEndDecode(&decoder);
 
   tCoderClear(&decoder);
@@ -1468,7 +1468,7 @@ int32_t tDeserializeSSyncDbReq(void *buf, int32_t bufLen, SSyncDbReq *pReq) {
 
 static int32_t tSerializeSUseDbRspImp(SCoder *pEncoder, SUseDbRsp *pRsp) {
   if (tEncodeCStr(pEncoder, pRsp->db) < 0) return -1;
-  if (tEncodeU64(pEncoder, pRsp->uid) < 0) return -1;
+  if (tEncodeI64(pEncoder, pRsp->uid) < 0) return -1;
   if (tEncodeI32(pEncoder, pRsp->vgVersion) < 0) return -1;
   if (tEncodeI32(pEncoder, pRsp->vgNum) < 0) return -1;
   if (tEncodeI8(pEncoder, pRsp->hashMethod) < 0) return -1;
@@ -1518,7 +1518,7 @@ int32_t tSerializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pRsp
 
 int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) {
   if (tDecodeCStrTo(pDecoder, pRsp->db) < 0) return -1;
-  if (tDecodeU64(pDecoder, &pRsp->uid) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pRsp->uid) < 0) return -1;
   if (tDecodeI32(pDecoder, &pRsp->vgVersion) < 0) return -1;
   if (tDecodeI32(pDecoder, &pRsp->vgNum) < 0) return -1;
   if (tDecodeI8(pDecoder, &pRsp->hashMethod) < 0) return -1;
@@ -1661,7 +1661,7 @@ static int32_t tEncodeSTableMetaRsp(SCoder *pEncoder, STableMetaRsp *pRsp) {
   if (tEncodeCStr(pEncoder, pRsp->tbName) < 0) return -1;
   if (tEncodeCStr(pEncoder, pRsp->stbName) < 0) return -1;
   if (tEncodeCStr(pEncoder, pRsp->dbFName) < 0) return -1;
-  if (tEncodeU64(pEncoder, pRsp->dbId) < 0) return -1;
+  if (tEncodeI64(pEncoder, pRsp->dbId) < 0) return -1;
   if (tEncodeI32(pEncoder, pRsp->numOfTags) < 0) return -1;
   if (tEncodeI32(pEncoder, pRsp->numOfColumns) < 0) return -1;
   if (tEncodeI8(pEncoder, pRsp->precision) < 0) return -1;
@@ -1684,7 +1684,7 @@ static int32_t tDecodeSTableMetaRsp(SCoder *pDecoder, STableMetaRsp *pRsp) {
   if (tDecodeCStrTo(pDecoder, pRsp->tbName) < 0) return -1;
   if (tDecodeCStrTo(pDecoder, pRsp->stbName) < 0) return -1;
   if (tDecodeCStrTo(pDecoder, pRsp->dbFName) < 0) return -1;
-  if (tDecodeU64(pDecoder, &pRsp->dbId) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pRsp->dbId) < 0) return -1;
   if (tDecodeI32(pDecoder, &pRsp->numOfTags) < 0) return -1;
   if (tDecodeI32(pDecoder, &pRsp->numOfColumns) < 0) return -1;
   if (tDecodeI8(pDecoder, &pRsp->precision) < 0) return -1;
@@ -2093,7 +2093,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
   if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
-  if (tEncodeU64(&encoder, pReq->dbUid) < 0) return -1;
+  if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->cacheBlockSize) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1;
@@ -2133,7 +2133,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
   if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
-  if (tDecodeU64(&decoder, &pReq->dbUid) < 0) return -1;
+  if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->cacheBlockSize) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1;
@@ -2171,7 +2171,7 @@ int32_t tSerializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq)
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
-  if (tEncodeU64(&encoder, pReq->dbUid) < 0) return -1;
+  if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
   tEndEncode(&encoder);
 
@@ -2187,7 +2187,7 @@ int32_t tDeserializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
-  if (tDecodeU64(&decoder, &pReq->dbUid) < 0) return -1;
+  if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
   tEndDecode(&decoder);
 
@@ -2356,35 +2356,7 @@ int32_t tDecodeSMqOffset(SCoder *decoder, SMqOffset *pOffset) {
   return 0;
 }
 
-int32_t tEncodeSMqVgOffsets(SCoder *encoder, const SMqVgOffsets *pOffsets) {
-  if (tStartEncode(encoder) < 0) return -1;
-  if (tEncodeI32(encoder, pOffsets->vgId) < 0) return -1;
-  int32_t sz = taosArrayGetSize(pOffsets->offsets);
-  if (tEncodeI32(encoder, sz) < 0) return -1;
-  for (int32_t i = 0; i < sz; i++) {
-    SMqOffset *offset = taosArrayGet(pOffsets->offsets, i);
-    if (tEncodeSMqOffset(encoder, offset) < 0) return -1;
-  }
-  tEndEncode(encoder);
-  return encoder->pos;
-}
-
-int32_t tDecodeSMqVgOffsets(SCoder *decoder, SMqVgOffsets *pOffsets) {
-  int32_t sz;
-  if (tStartDecode(decoder) < 0) return -1;
-  if (tDecodeI32(decoder, &pOffsets->vgId) < 0) return -1;
-  if (tDecodeI32(decoder, &sz) < 0) return -1;
-  pOffsets->offsets = taosArrayInit(sz, sizeof(SMqOffset));
-  for (int32_t i = 0; i < sz; i++) {
-    SMqOffset offset;
-    if (tDecodeSMqOffset(decoder, &offset) < 0) return -1;
-    taosArrayPush(pOffsets->offsets, &offset);
-  }
-  tEndDecode(decoder);
-  return 0;
-}
-
-int32_t tEncodeSMqCMResetOffsetReq(SCoder *encoder, const SMqCMResetOffsetReq *pReq) {
+int32_t tEncodeSMqCMCommitOffsetReq(SCoder *encoder, const SMqCMCommitOffsetReq *pReq) {
   if (tStartEncode(encoder) < 0) return -1;
   if (tEncodeI32(encoder, pReq->num) < 0) return -1;
   for (int32_t i = 0; i < pReq->num; i++) {
@@ -2394,7 +2366,7 @@ int32_t tEncodeSMqCMResetOffsetReq(SCoder *encoder, const SMqCMResetOffsetReq *p
   return encoder->pos;
 }
 
-int32_t tDecodeSMqCMResetOffsetReq(SCoder *decoder, SMqCMResetOffsetReq *pReq) {
+int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq) {
   if (tStartDecode(decoder) < 0) return -1;
   if (tDecodeI32(decoder, &pReq->num) < 0) return -1;
   pReq->offsets = TCODER_MALLOC(pReq->num * sizeof(SMqOffset), decoder);
@@ -2405,23 +2377,3 @@ int32_t tDecodeSMqCMResetOffsetReq(SCoder *decoder, SMqCMResetOffsetReq *pReq) {
   tEndDecode(decoder);
   return 0;
 }
-
-#if 0
-int32_t tEncodeSMqMVResetOffsetReq(SCoder *encoder, const SMqMVResetOffsetReq *pReq) {
-  if (tEncodeI64(encoder, pReq->leftForVer) < 0) return -1;
-  for (int32_t i = 0; i < pReq->num; i++) {
-    tEncodeSMqOffset(encoder, &pReq->offsets[i]);
-  }
-  return encoder->pos;
-}
-
-int32_t tDecodeSMqMVResetOffsetReq(SCoder *decoder, SMqMVResetOffsetReq *pReq) {
-  if (tDecodeI32(decoder, &pReq->num) < 0) return -1;
-  pReq->offsets = TCODER_MALLOC(pReq->num * sizeof(SMqOffset), decoder);
-  if (pReq->offsets == NULL) return -1;
-  for (int32_t i = 0; i < pReq->num; i++) {
-    tDecodeSMqOffset(decoder, &pReq->offsets[i]);
-  }
-  return 0;
-}
-#endif

source/common/src/tname.c

@@ -31,7 +31,7 @@ SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFil
   }
 
   assert(src->filterstr == 0 || src->filterstr == 1);
-  assert(!(src->lowerRelOptr == TSDB_RELATION_INVALID && src->upperRelOptr == TSDB_RELATION_INVALID));
+  assert(!(src->lowerRelOptr == 0 && src->upperRelOptr == 0));
 
   return pFilter;
 }

source/common/src/ttimezone.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#include "os.h"
-#include "ulog.h"
-#include "tglobal.h"
-#include "tconfig.h"
-#include "tutil.h"
-
-// TODO refactor to set the tz value through parameter
-void tsSetTimeZone() {
-  SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone");
-  if (cfg_timezone != NULL) {
-    uInfo("timezone is set to %s by %s", tsTimezone, tsCfgStatusStr[cfg_timezone->cfgStatus]);
-  }
-
-#ifdef WINDOWS
-  char winStr[TSDB_LOCALE_LEN * 2];
-  sprintf(winStr, "TZ=%s", tsTimezone);
-  putenv(winStr);
-#else
-  setenv("TZ", tsTimezone, 1);
-#endif
-  tzset();
-
-  /*
-  * get CURRENT time zone.
-  * system current time zone is affected by daylight saving time(DST)
-  *
-  * e.g., the local time zone of London in DST is GMT+01:00,
-  * otherwise is GMT+00:00
-  */
-#ifdef _MSC_VER
-#if _MSC_VER >= 1900
-  // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019
-  int64_t timezone = _timezone;
-  int32_t daylight = _daylight;
-  char **tzname = _tzname;
-#endif
-#endif
-
-  int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR);
-  tz += daylight;
-
-  /*
-  * format:
-  * (CST, +0800)
-  * (BST, +0100)
-  */
-  sprintf(tsTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz));
-  tsDaylight = daylight;
-
-  uInfo("timezone format changed to %s", tsTimezone);
-}

source/common/src/ttszip.c

@@ -23,7 +23,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) {
 
   pTSBuf->autoDelete = autoDelete;
   
-  taosGetTmpfilePath(tsTempDir, "join", pTSBuf->path);
+  taosGetTmpfilePath(osTempDir(), "join", pTSBuf->path);
   pTSBuf->f = fopen(pTSBuf->path, "wb+");
   if (pTSBuf->f == NULL) {
     free(pTSBuf);

source/common/src/ttypes.c

@@ -585,7 +585,7 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) {
 }
 
 void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) {
-  if (optr == TSDB_BINARY_OP_ADD) {
+  if (optr == OP_TYPE_ADD) {
     switch (type) {
       case TSDB_DATA_TYPE_TINYINT:
         *((int8_t *)dst) = GET_INT8_VAL(s1) + GET_INT8_VAL(s2);

source/dnode/mgmt/daemon/CMakeLists.txt

 aux_source_directory(src DAEMON_SRC)
 add_executable(taosd ${DAEMON_SRC})
-target_link_libraries(
+
+target_include_directories(
     taosd
-    PUBLIC dnode
-    PUBLIC util
-    PUBLIC os
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
+
+target_link_libraries(taosd dnode config util os)

source/dnode/mgmt/daemon/inc/dmnInt.h

+
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_DMN_INT_H_
+#define _TD_DMN_INT_H_
+
+#include "config.h"
+#include "dnode.h"
+#include "taoserror.h"
+#include "tglobal.h"
+#include "ulog.h"
+#include "version.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t dmnAddLogCfg(SConfig *pCfg);
+int32_t dmnInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl);
+int32_t dmnLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl);
+
+SConfig     *dmnReadCfg(const char *cfgDir, const char *envFile, const char *apolloUrl);
+SDnodeEnvCfg dmnGetEnvCfg(SConfig *pCfg);
+SDnodeObjCfg dmnGetObjCfg(SConfig *pCfg);
+
+void dmnDumpCfg(SConfig *pCfg);
+void dmnPrintVersion();
+void dmnGenerateGrant();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_DMN_INT_H_*/

source/dnode/mgmt/daemon/src/dmnCfg.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dmnInt.h"
+
+static int32_t dmnAddEpCfg(SConfig *pCfg) {
+  char defaultFqdn[TSDB_FQDN_LEN] = {0};
+  if (taosGetFqdn(defaultFqdn) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+  if (cfgAddString(pCfg, "fqdn", defaultFqdn) != 0) return -1;
+
+  int32_t defaultServerPort = 6030;
+  if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056) != 0) return -1;
+
+  char defaultFirstEp[TSDB_EP_LEN] = {0};
+  char defaultSecondEp[TSDB_EP_LEN] = {0};
+  snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort);
+  snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort);
+  if (cfgAddString(pCfg, "firstEp", defaultFirstEp) != 0) return -1;
+  if (cfgAddString(pCfg, "secondEp", defaultSecondEp) != 0) return -1;
+
+  return 0;
+}
+
+static int32_t dmnAddDirCfg(SConfig *pCfg) {
+  if (cfgAddDir(pCfg, "dataDir", osDataDir()) != 0) return -1;
+  if (cfgAddDir(pCfg, "tempDir", osTempDir()) != 0) return -1;
+  if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000) != 0) return -1;
+  if (cfgAddFloat(pCfg, "minimalTempDirGB", 1.0f, 0.001f, 10000000) != 0) return -1;
+  return 0;
+}
+
+static int32_t dmnCheckDirCfg(SConfig *pCfg) {
+  osSetDataDir(cfgGetItem(pCfg, "dataDir")->str);
+  osSetTempDir(cfgGetItem(pCfg, "tempDir")->str);
+  osSetTempReservedSpace(cfgGetItem(pCfg, "minimalDataDirGB")->fval);
+  osSetDataReservedSpace(cfgGetItem(pCfg, "minimalTempDirGB")->fval);
+  return 0;
+}
+
+static int32_t dmnAddVersionCfg(SConfig *pCfg) {
+  if (cfgAddString(pCfg, "buildinfo", buildinfo) != 0) return -1;
+  if (cfgAddString(pCfg, "gitinfo", gitinfo) != 0) return -1;
+  if (cfgAddString(pCfg, "version", version) != 0) return -1;
+  return 0;
+}
+
+static int32_t dmnAddDnodeCfg(SConfig *pCfg) {
+  if (dmnAddEpCfg(pCfg) != 0) return -1;
+  if (dmnAddDirCfg(pCfg) != 0) return -1;
+  if (dmnAddVersionCfg(pCfg) != 0) return -1;
+
+  if (cfgAddTimezone(pCfg, "timezone", "") != 0) return -1;
+  if (cfgAddLocale(pCfg, "locale", "") != 0) return -1;
+  if (cfgAddCharset(pCfg, "charset", "") != 0) return -1;
+  if (cfgAddInt32(pCfg, "numOfCores", 2, 1, 100000) != 0) return -1;
+  if (cfgAddInt32(pCfg, "numOfCommitThreads", 4, 1, 1000) != 0) return -1;
+  if (cfgAddBool(pCfg, "telemetryReporting", 0) != 0) return -1;
+  if (cfgAddBool(pCfg, "enableCoreFile", 0) != 0) return -1;
+  if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536) != 0) return -1;
+  if (cfgAddInt32(pCfg, "statusInterval", 1, 1, 30) != 0) return -1;
+  if (cfgAddFloat(pCfg, "numOfThreadsPerCore", 1, 0, 10) != 0) return -1;
+  if (cfgAddFloat(pCfg, "ratioOfQueryCores", 1, 0, 5) != 0) return -1;
+  if (cfgAddInt32(pCfg, "maxShellConns", 50000, 10, 50000000) != 0) return -1;
+  if (cfgAddInt32(pCfg, "shellActivityTimer", 3, 1, 120) != 0) return -1;
+  if (cfgAddInt32(pCfg, "rpcTimer", 300, 100, 3000) != 0) return -1;
+  if (cfgAddInt32(pCfg, "rpcMaxTime", 600, 100, 7200) != 0) return -1;
+  
+  return 0;
+}
+
+static void dmnSetDnodeCfg(SConfig *pCfg) {
+  SConfigItem *pItem = cfgGetItem(pCfg, "timezone");
+  osSetTimezone(pItem->str);
+  uDebug("timezone format changed from %s to %s", pItem->str, osTimezone());
+  cfgSetItem(pCfg, "timezone", osTimezone(), pItem->stype);
+}
+
+static int32_t dmnCheckCfg(SConfig *pCfg) {
+  bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval;
+  taosSetCoreDump(enableCore);
+
+  dmnSetDnodeCfg(pCfg);
+
+  if (dmnCheckDirCfg(pCfg) != 0) {
+    return -1;
+  }
+
+  taosGetSystemInfo();
+
+
+  if (tsNumOfCores <= 0) {
+    tsNumOfCores = 1;
+  }
+
+  if (tsQueryBufferSize >= 0) {
+    tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL;
+  }
+
+  return 0;
+}
+
+SConfig *dmnReadCfg(const char *cfgDir, const char *envFile, const char *apolloUrl) {
+  SConfig *pCfg = cfgInit();
+  if (pCfg == NULL) return NULL;
+
+  if (dmnAddLogCfg(pCfg) != 0) {
+    uError("failed to add log cfg since %s", terrstr());
+    cfgCleanup(pCfg);
+    return NULL;
+  }
+
+  if (dmnAddDnodeCfg(pCfg) != 0) {
+    uError("failed to init dnode cfg since %s", terrstr());
+    cfgCleanup(pCfg);
+    return NULL;
+  }
+
+  if (dmnLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) {
+    uError("failed to load cfg since %s", terrstr());
+    cfgCleanup(pCfg);
+    return NULL;
+  }
+
+  if (dmnCheckCfg(pCfg) != 0) {
+    uError("failed to check cfg since %s", terrstr());
+    cfgCleanup(pCfg);
+    return NULL;
+  }
+
+  cfgDumpCfg(pCfg);
+  return pCfg;
+}
+
+void dmnDumpCfg(SConfig *pCfg) {
+  printf("taos global config:\n");
+  printf("==================================\n");
+
+  SConfigItem *pItem = cfgIterate(pCfg, NULL);
+  while (pItem != NULL) {
+    switch (pItem->dtype) {
+      case CFG_DTYPE_BOOL:
+        printf("cfg:%s, value:%u src:%s\n", pItem->name, pItem->bval, cfgStypeStr(pItem->stype));
+        break;
+      case CFG_DTYPE_INT32:
+        printf("cfg:%s, value:%d src:%s\n", pItem->name, pItem->i32, cfgStypeStr(pItem->stype));
+        break;
+      case CFG_DTYPE_INT64:
+        printf("cfg:%s, value:%" PRId64 " src:%s\n", pItem->name, pItem->i64, cfgStypeStr(pItem->stype));
+        break;
+      case CFG_DTYPE_FLOAT:
+        printf("cfg:%s, value:%f src:%s\n", pItem->name, pItem->fval, cfgStypeStr(pItem->stype));
+        break;
+      case CFG_DTYPE_STRING:
+      case CFG_DTYPE_IPSTR:
+      case CFG_DTYPE_DIR:
+      case CFG_DTYPE_LOCALE:
+      case CFG_DTYPE_CHARSET:
+      case CFG_DTYPE_TIMEZONE:
+        printf("cfg:%s, value:%s src:%s\n", pItem->name, pItem->str, cfgStypeStr(pItem->stype));
+        break;
+    }
+    pItem = cfgIterate(pCfg, pItem);
+  }
+}
+
+SDnodeEnvCfg dmnGetEnvCfg(SConfig *pCfg) {
+  SDnodeEnvCfg envCfg = {0};
+
+  const char *vstr = cfgGetItem(pCfg, "version")->str;
+  envCfg.sver = 30000000;
+  tstrncpy(envCfg.buildinfo, cfgGetItem(pCfg, "buildinfo")->str, sizeof(envCfg.buildinfo));
+  tstrncpy(envCfg.gitinfo, cfgGetItem(pCfg, "gitinfo")->str, sizeof(envCfg.gitinfo));
+  tstrncpy(envCfg.timezone, cfgGetItem(pCfg, "timezone")->str, sizeof(envCfg.timezone));
+  tstrncpy(envCfg.locale, cfgGetItem(pCfg, "locale")->str, sizeof(envCfg.locale));
+  tstrncpy(envCfg.charset, cfgGetItem(pCfg, "charset")->str, sizeof(envCfg.charset));
+  envCfg.numOfCores = cfgGetItem(pCfg, "numOfCores")->i32;
+  envCfg.numOfCommitThreads = (uint16_t)cfgGetItem(pCfg, "numOfCommitThreads")->i32;
+  envCfg.enableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval;
+  envCfg.rpcMaxTime = cfgGetItem(pCfg, "rpcMaxTime")->i32;
+  envCfg.rpcTimer = cfgGetItem(pCfg, "rpcTimer")->i32;
+
+  return envCfg;
+}
+
+SDnodeObjCfg dmnGetObjCfg(SConfig *pCfg) {
+  SDnodeObjCfg objCfg = {0};
+
+  objCfg.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32;
+  objCfg.statusInterval = cfgGetItem(pCfg, "statusInterval")->i32;
+  objCfg.numOfThreadsPerCore = cfgGetItem(pCfg, "numOfThreadsPerCore")->fval;
+  objCfg.ratioOfQueryCores = cfgGetItem(pCfg, "ratioOfQueryCores")->fval;
+  objCfg.maxShellConns = cfgGetItem(pCfg, "maxShellConns")->i32;
+  objCfg.shellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32;
+  tstrncpy(objCfg.dataDir, cfgGetItem(pCfg, "dataDir")->str, sizeof(objCfg.dataDir));
+
+  tstrncpy(objCfg.firstEp, cfgGetItem(pCfg, "firstEp")->str, sizeof(objCfg.firstEp));
+  tstrncpy(objCfg.secondEp, cfgGetItem(pCfg, "secondEp")->str, sizeof(objCfg.firstEp));
+  objCfg.serverPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32;
+  tstrncpy(objCfg.localFqdn, cfgGetItem(pCfg, "fqdn")->str, sizeof(objCfg.localFqdn));
+  snprintf(objCfg.localEp, sizeof(objCfg.localEp), "%s:%u", objCfg.localFqdn, objCfg.serverPort);
+  return objCfg;
+}
\ No newline at end of file

source/dnode/mgmt/daemon/src/dmnLog.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dmnInt.h"
+
+int32_t dmnAddLogCfg(SConfig *pCfg) {
+  if (cfgAddDir(pCfg, "logDir", osLogDir()) != 0) return -1;
+  if (cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000) != 0) return -1;
+  if (cfgAddBool(pCfg, "asyncLog", 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "numOfLogLines", 10000000, 1000, 2000000000) != 0) return -1;
+  if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000) != 0) return -1;
+  if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "dDebugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "vDebugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "mDebugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "cDebugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "jniDebugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "tmrDebugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "uDebugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "rpcDebugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "qDebugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "wDebugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "sDebugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "tsdbDebugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "tqDebugFlag", 0, 0, 255) != 0) return -1;
+  if (cfgAddInt32(pCfg, "fsDebugFlag", 0, 0, 255) != 0) return -1;
+  return 0;
+}
+
+int32_t dmnSetLogCfg(SConfig *pCfg) {
+  osSetLogDir(cfgGetItem(pCfg, "logDir")->str);
+  osSetLogReservedSpace(cfgGetItem(pCfg, "minimalLogDirGB")->fval);
+  tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval;
+  tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32;
+  tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32;
+  dDebugFlag = cfgGetItem(pCfg, "dDebugFlag")->i32;
+  vDebugFlag = cfgGetItem(pCfg, "vDebugFlag")->i32;
+  mDebugFlag = cfgGetItem(pCfg, "mDebugFlag")->i32;
+  cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32;
+  jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32;
+  tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32;
+  uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32;
+  rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32;
+  qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32;
+  wDebugFlag = cfgGetItem(pCfg, "wDebugFlag")->i32;
+  sDebugFlag = cfgGetItem(pCfg, "sDebugFlag")->i32;
+  tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32;
+  tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32;
+  fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32;
+
+  int32_t debugFlag = cfgGetItem(pCfg, "debugFlag")->i32;
+  taosSetAllDebugFlag(debugFlag);
+
+  return 0;
+}
+
+int32_t dmnInitLog(const char *cfgDir, const char *envFile, const char *apolloUrl) {
+  SConfig *pCfg = cfgInit();
+  if (pCfg == NULL) return -1;
+
+  if (dmnAddLogCfg(pCfg) != 0) {
+    printf("failed to add log cfg since %s\n", terrstr());
+    cfgCleanup(pCfg);
+    return -1;
+  }
+
+  if (dmnLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) {
+    printf("failed to load log cfg since %s\n", terrstr());
+    cfgCleanup(pCfg);
+    return -1;
+  }
+
+  if (dmnSetLogCfg(pCfg) != 0) {
+    printf("failed to set log cfg since %s\n", terrstr());
+    cfgCleanup(pCfg);
+    return -1;
+  }
+
+  if (taosInitLog("taosdlog", 1) != 0) {
+    printf("failed to init log file since %s\n", terrstr());
+    cfgCleanup(pCfg);
+    return -1;
+  }
+
+  cfgCleanup(pCfg);
+  return 0;
+}
+
+int32_t dmnLoadCfg(SConfig *pConfig, const char *inputCfgDir, const char *envFile, const char *apolloUrl) {
+  char configDir[PATH_MAX] = {0};
+  char configFile[PATH_MAX + 100] = {0};
+
+  taosExpandDir(inputCfgDir, configDir, PATH_MAX);
+  snprintf(configFile, sizeof(configFile), "%s" TD_DIRSEP "taos.cfg", configDir);
+
+  if (cfgLoad(pConfig, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) {
+    uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr());
+    return -1;
+  }
+
+  if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, configFile) != 0) {
+    if (cfgLoad(pConfig, CFG_STYPE_CFG_FILE, configDir) != 0) {
+      uError("failed to load from config file:%s since %s\n", configFile, terrstr());
+      return -1;
+    }
+  }
+
+  if (cfgLoad(pConfig, CFG_STYPE_ENV_FILE, envFile) != 0) {
+    uError("failed to load from env file:%s since %s\n", envFile, terrstr());
+    return -1;
+  }
+
+  if (cfgLoad(pConfig, CFG_STYPE_ENV_VAR, NULL) != 0) {
+    uError("failed to load from global env variables since %s\n", terrstr());
+    return -1;
+  }
+
+  return 0;
+}

source/dnode/mgmt/daemon/src/daemon.c → source/dnode/mgmt/daemon/src/dmnMain.c

@@ -14,12 +14,7 @@
  */
 
 #define _DEFAULT_SOURCE
-#include "dnode.h"
-#include "os.h"
-#include "tconfig.h"
-#include "tglobal.h"
-#include "tnote.h"
-#include "ulog.h"
+#include "dmnInt.h"
 
 static struct {
   bool stop;
@@ -27,15 +22,16 @@ static struct {
   bool generateGrant;
   bool printAuth;
   bool printVersion;
-  char configDir[PATH_MAX];
+  char envFile[PATH_MAX];
+  char apolloUrl[PATH_MAX];
 } dmn = {0};
 
-void dmnSigintHandle(int signum, void *info, void *ctx) {
+static void dmnSigintHandle(int signum, void *info, void *ctx) {
   uInfo("singal:%d is received", signum);
   dmn.stop = true;
 }
 
-void dmnSetSignalHandle() {
+static void dmnSetSignalHandle() {
   taosSetSignal(SIGTERM, dmnSigintHandle);
   taosSetSignal(SIGHUP, dmnSigintHandle);
   taosSetSignal(SIGINT, dmnSigintHandle);
@@ -43,17 +39,22 @@ void dmnSetSignalHandle() {
   taosSetSignal(SIGBREAK, dmnSigintHandle);
 }
 
-int dmnParseOption(int argc, char const *argv[]) {
-  tstrncpy(dmn.configDir, "/etc/taos", PATH_MAX);
+static void dmnWaitSignal() {
+  dmnSetSignalHandle();
+  while (!dmn.stop) {
+    taosMsleep(100);
+  }
+}
 
-  for (int i = 1; i < argc; ++i) {
+static int32_t dmnParseOption(int32_t argc, char const *argv[]) {
+  for (int32_t i = 1; i < argc; ++i) {
     if (strcmp(argv[i], "-c") == 0) {
       if (i < argc - 1) {
         if (strlen(argv[++i]) >= PATH_MAX) {
           printf("config file path overflow");
           return -1;
         }
-        tstrncpy(dmn.configDir, argv[i], PATH_MAX);
+        tstrncpy(configDir, argv[i], PATH_MAX);
       } else {
         printf("'-c' requires a parameter, default is %s\n", configDir);
         return -1;
@@ -62,8 +63,6 @@ int dmnParseOption(int argc, char const *argv[]) {
       dmn.dumpConfig = true;
     } else if (strcmp(argv[i], "-k") == 0) {
       dmn.generateGrant = true;
-    } else if (strcmp(argv[i], "-A") == 0) {
-      dmn.printAuth = true;
     } else if (strcmp(argv[i], "-V") == 0) {
       dmn.printVersion = true;
     } else {
@@ -73,110 +72,17 @@ int dmnParseOption(int argc, char const *argv[]) {
   return 0;
 }
 
-void dmnGenerateGrant() {
-#if 0
-  grantParseParameter();
-#endif
-}
-
-void dmnPrintVersion() {
-#ifdef TD_ENTERPRISE
-  char *releaseName = "enterprise";
-#else
-  char *releaseName = "community";
-#endif
-  printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version);
-  printf("gitinfo: %s\n", gitinfo);
-  printf("gitinfoI: %s\n", gitinfoOfInternal);
-  printf("builuInfo: %s\n", buildinfo);
-}
-
-int dmnReadConfig(const char *path) {
-  tstrncpy(configDir, dmn.configDir, PATH_MAX);
-  taosInitGlobalCfg();
-  taosReadGlobalLogCfg();
-
-  if (taosMkDir(tsLogDir) != 0) {
-    printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno));
-    return -1;
-  }
-
-  char temp[PATH_MAX];
-  snprintf(temp, PATH_MAX, "%s/taosdlog", tsLogDir);
-  if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) {
-    printf("failed to init log file\n");
-    return -1;
-  }
-
-  if (taosInitNotes() != 0) {
-    printf("failed to init log file\n");
-    return -1;
-  }
-
-  if (taosReadCfgFromFile() != 0) {
-    uError("failed to read config");
-    return -1;
-  }
-
-  if (taosCheckAndPrintCfg() != 0) {
-    uError("failed to check config");
-    return -1;
-  }
-
-  taosSetCoreDump(tsEnableCoreFile);
-  return 0;
-}
-
-void dmnDumpConfig() { taosDumpGlobalCfg(); }
-
-void dmnWaitSignal() {
-  dmnSetSignalHandle();
-  while (!dmn.stop) {
-    taosMsleep(100);
-  }
-}
-
-void dnmInitEnvCfg(SDnodeEnvCfg *pCfg) {
-  pCfg->sver = 30000000;  // 3.0.0.0
-  pCfg->numOfCores = tsNumOfCores;
-  pCfg->numOfCommitThreads = tsNumOfCommitThreads;
-  pCfg->enableTelem = 0;
-  tstrncpy(pCfg->timezone, tsTimezone, TSDB_TIMEZONE_LEN);
-  tstrncpy(pCfg->locale, tsLocale, TSDB_LOCALE_LEN);
-  tstrncpy(pCfg->charset, tsCharset, TSDB_LOCALE_LEN);
-  tstrncpy(pCfg->buildinfo, buildinfo, 64);
-  tstrncpy(pCfg->gitinfo, gitinfo, 48);
-}
-
-void dmnInitObjCfg(SDnodeObjCfg *pCfg) {
-  pCfg->numOfSupportVnodes = tsNumOfSupportVnodes;
-  pCfg->statusInterval = tsStatusInterval;
-  pCfg->numOfThreadsPerCore = tsNumOfThreadsPerCore;
-  pCfg->ratioOfQueryCores = tsRatioOfQueryCores;
-  pCfg->maxShellConns = tsMaxShellConns;
-  pCfg->shellActivityTimer = tsShellActivityTimer;
-  pCfg->serverPort = tsServerPort;
-  tstrncpy(pCfg->dataDir, tsDataDir, TSDB_FILENAME_LEN);
-  tstrncpy(pCfg->localEp, tsLocalEp, TSDB_EP_LEN);
-  tstrncpy(pCfg->localFqdn, tsLocalFqdn, TSDB_FQDN_LEN);
-  tstrncpy(pCfg->firstEp, tsFirst, TSDB_EP_LEN);
-}
-
-int dmnRunDnode() {
-  SDnodeEnvCfg envCfg = {0};
-  SDnodeObjCfg objCfg = {0};
-
-  dnmInitEnvCfg(&envCfg);
-  dmnInitObjCfg(&objCfg);
-
+int32_t dmnRunDnode(SConfig *pCfg) {
+  SDnodeEnvCfg envCfg = dmnGetEnvCfg(pCfg);
   if (dndInit(&envCfg) != 0) {
-    uInfo("Failed to start TDengine, please check the log at %s", tsLogDir);
+    uInfo("Failed to start TDengine, please check the log");
     return -1;
   }
 
+  SDnodeObjCfg objCfg = dmnGetObjCfg(pCfg);
   SDnode *pDnode = dndCreate(&objCfg);
   if (pDnode == NULL) {
-    uInfo("Failed to start TDengine, please check the log at %s", tsLogDir);
+    uInfo("Failed to start TDengine, please check the log");
     return -1;
   }
 
@@ -191,6 +97,8 @@ int dmnRunDnode() {
 }
 
 int main(int argc, char const *argv[]) {
+  osInit();
+
   if (dmnParseOption(argc, argv) != 0) {
     return -1;
   }
@@ -205,14 +113,23 @@ int main(int argc, char const *argv[]) {
     return 0;
   }
 
-  if (dmnReadConfig(dmn.configDir) != 0) {
+  if (dmnInitLog(configDir, dmn.envFile, dmn.apolloUrl) != 0) {
+    return -1;
+  }
+
+  SConfig *pCfg = dmnReadCfg(configDir, dmn.envFile, dmn.apolloUrl);
+  if (pCfg == NULL) {
+    uInfo("Failed to start TDengine since read config error");
     return -1;
   }
 
   if (dmn.dumpConfig) {
-    dmnDumpConfig();
+    dmnDumpCfg(pCfg);
+    cfgCleanup(pCfg);
     return 0;
   }
 
-  return dmnRunDnode();
+  int32_t code = dmnRunDnode(pCfg);
+  cfgCleanup(pCfg);
+  return code;
 }

source/dnode/mgmt/daemon/src/dmnUtil.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dmnInt.h"
+
+void dmnGenerateGrant() {
+#if 0
+  grantParseParameter();
+#endif
+}
+
+void dmnPrintVersion() {
+#ifdef TD_ENTERPRISE
+  char *releaseName = "enterprise";
+#else
+  char *releaseName = "community";
+#endif
+  printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version);
+  printf("gitinfo: %s\n", gitinfo);
+  printf("gitinfoI: %s\n", gitinfoOfInternal);
+  printf("builuInfo: %s\n", buildinfo);
+}

source/dnode/mgmt/impl/src/dndEnv.c

@@ -270,7 +270,8 @@ int32_t dndInit(const SDnodeEnvCfg *pCfg) {
   taosBlockSIGPIPE();
   taosResolveCRC();
 
-  if (rpcInit() != 0) {
+  SRpcCfg rpcCfg = {.rpcTimer = pCfg->rpcTimer, .rpcMaxTime = pCfg->rpcMaxTime, .sver = pCfg->sver};
+  if (rpcInit(&rpcCfg) != 0) {
     dError("failed to init rpc since %s", terrstr());
     dndCleanup();
     return -1;
@@ -325,18 +326,6 @@ void taosGetDisk() {
   SDiskSize    diskSize = tfsGetSize(pTfs);
   
   tfsUpdateSize(&fsMeta);
-  tsTotalDataDirGB = (float)(fsMeta.total / unit);
-  tsUsedDataDirGB = (float)(fsMeta.used / unit);
-  tsAvailDataDirGB = (float)(fsMeta.avail / unit);
 
-  if (taosGetDiskSize(tsLogDir, &diskSize) == 0) {
-    tsTotalLogDirGB = (float)(diskSize.total / unit);
-    tsAvailLogDirGB = (float)(diskSize.avail / unit);
-  }
-
-  if (taosGetDiskSize(tsTempDir, &diskSize) == 0) {
-    tsTotalTmpDirGB = (float)(diskSize.total / unit);
-    tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit);
-  }
 #endif
 }
\ No newline at end of file

source/dnode/mgmt/impl/src/dndMgmt.c

@@ -296,7 +296,7 @@ PRASE_DNODE_OVER:
   if (taosArrayGetSize(pMgmt->pDnodeEps) == 0) {
     SDnodeEp dnodeEp = {0};
     dnodeEp.isMnode = 1;
-    taosGetFqdnPortFromEp(pDnode->cfg.firstEp, &dnodeEp.ep);
+    taosGetFqdnPortFromEp(pDnode->cfg.firstEp, pDnode->cfg.serverPort, &dnodeEp.ep);
     taosArrayPush(pMgmt->pDnodeEps, &dnodeEp);
   }
 
@@ -371,9 +371,9 @@ void dndSendStatusReq(SDnode *pDnode) {
   req.clusterCfg.checkTime = 0;
   char timestr[32] = "1970-01-01 00:00:00.00";
   (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
-  memcpy(req.clusterCfg.timezone, pDnode->env.timezone, TSDB_TIMEZONE_LEN);
-  memcpy(req.clusterCfg.locale, pDnode->env.locale, TSDB_LOCALE_LEN);
-  memcpy(req.clusterCfg.charset, pDnode->env.charset, TSDB_LOCALE_LEN);
+  memcpy(req.clusterCfg.timezone, pDnode->env.timezone, TD_TIMEZONE_LEN);
+  memcpy(req.clusterCfg.locale, pDnode->env.locale, TD_LOCALE_LEN);
+  memcpy(req.clusterCfg.charset, pDnode->env.charset, TD_LOCALE_LEN);
   taosRUnLockLatch(&pMgmt->latch);
 
   req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad));

source/dnode/mgmt/impl/src/dndTransport.c

@@ -113,6 +113,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
   pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_TOPIC)] = dndProcessMnodeWriteMsg;
   pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_TOPIC)] = dndProcessMnodeWriteMsg;
   pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SUBSCRIBE)] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_MQ_COMMIT_OFFSET)] = dndProcessMnodeWriteMsg;
   /*pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBSCRIBE_RSP)] = dndProcessMnodeWriteMsg;*/
   pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN_RSP)] = dndProcessMnodeWriteMsg;
   pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB_RSP)] = dndProcessMnodeWriteMsg;
@@ -155,7 +156,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
 }
 
 static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) {
-  SDnode *    pDnode = parent;
+  SDnode     *pDnode = parent;
   STransMgmt *pMgmt = &pDnode->tmgmt;
 
   tmsg_t msgType = pRsp->msgType;
@@ -219,7 +220,7 @@ static void dndCleanupClient(SDnode *pDnode) {
 }
 
 static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) {
-  SDnode *    pDnode = param;
+  SDnode     *pDnode = param;
   STransMgmt *pMgmt = &pDnode->tmgmt;
 
   tmsg_t msgType = pReq->msgType;
@@ -313,7 +314,7 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char
   SAuthReq authReq = {0};
   tstrncpy(authReq.user, user, TSDB_USER_LEN);
   int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq);
-  void *  pReq = rpcMallocCont(contLen);
+  void   *pReq = rpcMallocCont(contLen);
   tSerializeSAuthReq(pReq, contLen, &authReq);
 
   SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528};

source/dnode/mgmt/impl/test/sut/inc/sut.h

@@ -20,11 +20,10 @@
 #include "os.h"
 
 #include "dnode.h"
-#include "tconfig.h"
+#include "tmsg.h"
 #include "tdataformat.h"
 #include "tglobal.h"
 #include "tmsg.h"
-#include "tnote.h"
 #include "trpc.h"
 #include "tthread.h"
 #include "ulog.h"

source/dnode/mgmt/impl/test/sut/src/sut.cpp

@@ -28,16 +28,13 @@ void Testbase::InitLog(const char* path) {
   wDebugFlag = 0;
   sDebugFlag = 0;
   tsdbDebugFlag = 0;
-  cqDebugFlag = 0;
   tscEmbeddedInUtil = 1;
   tsAsyncLog = 0;
 
   taosRemoveDir(path);
   taosMkDir(path);
-
-  char temp[PATH_MAX];
-  snprintf(temp, PATH_MAX, "%s/taosdlog", path);
-  if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) {
+  osSetLogDir(path);
+  if (taosInitLog("taosdlog", 1) != 0) {
     printf("failed to init log file\n");
   }
 }
@@ -46,6 +43,8 @@ void Testbase::Init(const char* path, int16_t port) {
   SDnodeEnvCfg cfg = {0};
   cfg.numOfCommitThreads = 1;
   cfg.numOfCores = 1;
+  cfg.rpcMaxTime = 600;
+  cfg.rpcTimer = 300;
   dndInit(&cfg);
 
   char fqdn[] = "localhost";

source/dnode/mnode/impl/inc/mndDef.h

@@ -123,6 +123,7 @@ typedef enum {
   TRN_TYPE_DROP_TOPIC = 1015,
   TRN_TYPE_SUBSCRIBE = 1016,
   TRN_TYPE_REBALANCE = 1017,
+  TRN_TYPE_COMMIT_OFFSET = 1018,
   TRN_TYPE_BASIC_SCOPE_END,
   TRN_TYPE_GLOBAL_SCOPE = 2000,
   TRN_TYPE_CREATE_DNODE = 2001,
@@ -176,7 +177,7 @@ typedef struct {
   SArray*    undoActions;
   int64_t    createdTime;
   int64_t    lastExecTime;
-  uint64_t   dbUid;
+  int64_t    dbUid;
   char       dbname[TSDB_DB_FNAME_LEN];
   char       lastError[TSDB_TRANS_ERROR_LEN];
 } STrans;
@@ -309,7 +310,7 @@ typedef struct {
   char    createUser[TSDB_USER_LEN];
   int64_t createdTime;
   int64_t updateTime;
-  uint64_t uid;
+  int64_t uid;
   int32_t cfgVersion;
   int32_t vgVersion;
   int8_t  hashMethod;  // default is 1
@@ -346,8 +347,8 @@ typedef struct {
   char     db[TSDB_DB_FNAME_LEN];
   int64_t  createdTime;
   int64_t  updateTime;
-  uint64_t uid;
-  uint64_t dbUid;
+  int64_t  uid;
+  int64_t  dbUid;
   int32_t  version;
   int32_t  nextColId;
   int32_t  numOfColumns;
@@ -465,6 +466,24 @@ static FORCE_INLINE void tDeleteSMqSubConsumer(SMqSubConsumer* pSubConsumer) {
   }
 }
 
+typedef struct {
+  char    key[TSDB_PARTITION_KEY_LEN];
+  int64_t offset;
+} SMqOffsetObj;
+
+static FORCE_INLINE int32_t tEncodeSMqOffsetObj(void** buf, const SMqOffsetObj* pOffset) {
+  int32_t tlen = 0;
+  tlen += taosEncodeString(buf, pOffset->key);
+  tlen += taosEncodeFixedI64(buf, pOffset->offset);
+  return tlen;
+}
+
+static FORCE_INLINE void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset) {
+  buf = taosDecodeStringTo(buf, pOffset->key);
+  buf = taosDecodeFixedI64(buf, &pOffset->offset);
+  return buf;
+}
+
 typedef struct {
   char    key[TSDB_SUBSCRIBE_KEY_LEN];
   int32_t status;

source/dnode/mnode/impl/inc/mndInt.h

@@ -56,11 +56,8 @@ typedef struct {
 } SProfileMgmt;
 
 typedef struct {
-  int8_t           enable;
-  pthread_mutex_t  lock;
-  pthread_cond_t   cond;
-  volatile int32_t exit;
-  pthread_t        thread;
+  bool     enable;
+  SRWLatch lock;
   char     email[TSDB_FQDN_LEN];
 } STelemMgmt;
 
@@ -81,6 +78,7 @@ typedef struct SMnode {
   tmr_h             timer;
   tmr_h             transTimer;
   tmr_h             mqTimer;
+  tmr_h             telemTimer;
   char             *path;
   SMnodeCfg         cfg;
   int64_t           checkTime;

source/dnode/mnode/impl/inc/mndOffset.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_MND_OFFSET_H_
+#define _TD_MND_OFFSET_H_
+
+#include "mndInt.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t mndInitOffset(SMnode *pMnode);
+void    mndCleanupOffset(SMnode *pMnode);
+
+SMqOffsetObj *mndAcquireOffset(SMnode *pMnode, const char *key);
+void          mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset);
+
+SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset);
+SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw);
+
+int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs);
+
+static FORCE_INLINE int32_t mndMakePartitionKey(char *key, const char *cgroup, const char *topicName, int32_t vgId) {
+  return snprintf(key, TSDB_PARTITION_KEY_LEN, "%d:%s:%s", vgId, cgroup, topicName);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_MND_OFFSET_H_*/

source/dnode/mnode/impl/src/mndCluster.c

@@ -52,7 +52,7 @@ int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) {
   SSdb *pSdb = pMnode->pSdb;
 
   SClusterObj *pCluster = sdbAcquire(pSdb, SDB_CLUSTER, &pMnode->clusterId);
-  if (pCluster = NULL) {
+  if (pCluster == NULL) {
     return -1;
   }
 

source/dnode/mnode/impl/src/mndOffset.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "mndOffset.h"
+#include "mndAuth.h"
+#include "mndDb.h"
+#include "mndDnode.h"
+#include "mndMnode.h"
+#include "mndShow.h"
+#include "mndStb.h"
+#include "mndTrans.h"
+#include "mndUser.h"
+#include "mndVgroup.h"
+#include "tname.h"
+
+#define MND_OFFSET_VER_NUMBER   1
+#define MND_OFFSET_RESERVE_SIZE 64
+
+static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset);
+static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset);
+static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOffset, SMqOffsetObj *pNewOffset);
+static int32_t mndProcessCommitOffsetReq(SMnodeMsg *pReq);
+
+int32_t mndInitOffset(SMnode *pMnode) {
+  SSdbTable table = {.sdbType = SDB_OFFSET,
+                     .keyType = SDB_KEY_BINARY,
+                     .encodeFp = (SdbEncodeFp)mndOffsetActionEncode,
+                     .decodeFp = (SdbDecodeFp)mndOffsetActionDecode,
+                     .insertFp = (SdbInsertFp)mndOffsetActionInsert,
+                     .updateFp = (SdbUpdateFp)mndOffsetActionUpdate,
+                     .deleteFp = (SdbDeleteFp)mndOffsetActionDelete};
+
+  mndSetMsgHandle(pMnode, TDMT_MND_MQ_COMMIT_OFFSET, mndProcessCommitOffsetReq);
+
+  return sdbSetTable(pMnode->pSdb, table);
+}
+
+void mndCleanupOffset(SMnode *pMnode) {}
+
+SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset) {
+  terrno = TSDB_CODE_OUT_OF_MEMORY;
+  void   *buf = NULL;
+  int32_t tlen = tEncodeSMqOffsetObj(NULL, pOffset);
+  int32_t size = sizeof(int32_t) + tlen + MND_OFFSET_RESERVE_SIZE;
+
+  SSdbRaw *pRaw = sdbAllocRaw(SDB_OFFSET, MND_OFFSET_VER_NUMBER, size);
+  if (pRaw == NULL) goto OFFSET_ENCODE_OVER;
+
+  buf = malloc(tlen);
+  if (buf == NULL) goto OFFSET_ENCODE_OVER;
+
+  void *abuf = buf;
+  tEncodeSMqOffsetObj(&abuf, pOffset);
+
+  int32_t dataPos = 0;
+  SDB_SET_INT32(pRaw, dataPos, tlen, OFFSET_ENCODE_OVER);
+  SDB_SET_BINARY(pRaw, dataPos, buf, tlen, OFFSET_ENCODE_OVER);
+  SDB_SET_RESERVE(pRaw, dataPos, MND_OFFSET_RESERVE_SIZE, OFFSET_ENCODE_OVER);
+  SDB_SET_DATALEN(pRaw, dataPos, OFFSET_ENCODE_OVER);
+
+  terrno = TSDB_CODE_SUCCESS;
+
+OFFSET_ENCODE_OVER:
+  tfree(buf);
+  if (terrno != TSDB_CODE_SUCCESS) {
+    mError("offset:%s, failed to encode to raw:%p since %s", pOffset->key, pRaw, terrstr());
+    sdbFreeRaw(pRaw);
+    return NULL;
+  }
+
+  mTrace("offset:%s, encode to raw:%p, row:%p", pOffset->key, pRaw, pOffset);
+  return pRaw;
+}
+
+SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw) {
+  terrno = TSDB_CODE_OUT_OF_MEMORY;
+  void *buf = NULL;
+
+  int8_t sver = 0;
+  if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto OFFSET_DECODE_OVER;
+
+  if (sver != MND_OFFSET_VER_NUMBER) {
+    terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
+    goto OFFSET_DECODE_OVER;
+  }
+
+  int32_t  size = sizeof(SMqOffsetObj);
+  SSdbRow *pRow = sdbAllocRow(size);
+  if (pRow == NULL) goto OFFSET_DECODE_OVER;
+
+  SMqOffsetObj *pOffset = sdbGetRowObj(pRow);
+  if (pOffset == NULL) goto OFFSET_DECODE_OVER;
+
+  int32_t dataPos = 0;
+  int32_t tlen;
+  SDB_GET_INT32(pRaw, dataPos, &tlen, OFFSET_DECODE_OVER);
+  buf = malloc(tlen + 1);
+  if (buf == NULL) goto OFFSET_DECODE_OVER;
+  SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OFFSET_DECODE_OVER);
+  SDB_GET_RESERVE(pRaw, dataPos, MND_OFFSET_RESERVE_SIZE, OFFSET_DECODE_OVER);
+
+  if (tDecodeSMqOffsetObj(buf, pOffset) == NULL) {
+    goto OFFSET_DECODE_OVER;
+  }
+
+  terrno = TSDB_CODE_SUCCESS;
+
+OFFSET_DECODE_OVER:
+  tfree(buf);
+  if (terrno != TSDB_CODE_SUCCESS) {
+    mError("offset:%s, failed to decode from raw:%p since %s", pOffset->key, pRaw, terrstr());
+    tfree(pRow);
+    return NULL;
+  }
+
+  mTrace("offset:%s, decode from raw:%p, row:%p", pOffset->key, pRaw, pOffset);
+  return pRow;
+}
+
+int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs) {
+  int32_t code = 0;
+  int32_t sz = taosArrayGetSize(vgs);
+  for (int32_t i = 0; i < sz; i++) {
+    SMqConsumerEp *pConsumerEp = taosArrayGet(vgs, i);
+    SMqOffsetObj   offsetObj;
+    if (mndMakePartitionKey(offsetObj.key, cgroup, topicName, pConsumerEp->vgId) < 0) {
+      return -1;
+    }
+    offsetObj.offset = -1;
+    SSdbRaw *pOffsetRaw = mndOffsetActionEncode(&offsetObj);
+    if (pOffsetRaw == NULL) {
+      return -1;
+    }
+    sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY);
+    if (mndTransAppendRedolog(pTrans, pOffsetRaw) < 0) {
+      return -1;
+    }
+  }
+  return 0;
+}
+
+static int32_t mndProcessCommitOffsetReq(SMnodeMsg *pMsg) {
+  char key[TSDB_PARTITION_KEY_LEN];
+
+  SMnode              *pMnode = pMsg->pMnode;
+  char                *msgStr = pMsg->rpcMsg.pCont;
+  SMqCMCommitOffsetReq commitOffsetReq;
+  SCoder               decoder;
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, msgStr, pMsg->rpcMsg.contLen, TD_DECODER);
+
+  tDecodeSMqCMCommitOffsetReq(&decoder, &commitOffsetReq);
+
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_COMMIT_OFFSET, &pMsg->rpcMsg);
+
+  for (int32_t i = 0; i < commitOffsetReq.num; i++) {
+    SMqOffset *pOffset = &commitOffsetReq.offsets[i];
+    if (mndMakePartitionKey(key, pOffset->cgroup, pOffset->topicName, pOffset->vgId) < 0) {
+      return -1;
+    }
+    SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, key);
+    ASSERT(pOffsetObj);
+    pOffsetObj->offset = pOffset->offset;
+    SSdbRaw *pOffsetRaw = mndOffsetActionEncode(pOffsetObj);
+    sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY);
+    mndTransAppendRedolog(pTrans, pOffsetRaw);
+    mndReleaseOffset(pMnode, pOffsetObj);
+  }
+
+  if (mndTransPrepare(pMnode, pTrans) != 0) {
+    mError("mq-commit-offset-trans:%d, failed to prepare since %s", pTrans->id, terrstr());
+    mndTransDrop(pTrans);
+    return -1;
+  }
+
+  mndTransDrop(pTrans);
+  return TSDB_CODE_MND_ACTION_IN_PROGRESS;
+}
+
+static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset) {
+  mTrace("offset:%s, perform insert action", pOffset->key);
+  return 0;
+}
+
+static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset) {
+  mTrace("offset:%s, perform delete action", pOffset->key);
+  return 0;
+}
+
+static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOldOffset, SMqOffsetObj *pNewOffset) {
+  mTrace("offset:%s, perform update action", pOldOffset->key);
+  return 0;
+}
+
+SMqOffsetObj *mndAcquireOffset(SMnode *pMnode, const char *key) {
+  SSdb         *pSdb = pMnode->pSdb;
+  SMqOffsetObj *pOffset = sdbAcquire(pSdb, SDB_OFFSET, key);
+  if (pOffset == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
+    terrno = TSDB_CODE_MND_OFFSET_NOT_EXIST;
+  }
+  return pOffset;
+}
+
+void mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset) {
+  SSdb *pSdb = pMnode->pSdb;
+  sdbRelease(pSdb, pOffset);
+}
+
+static void mndCancelGetNextOffset(SMnode *pMnode, void *pIter) {
+  SSdb *pSdb = pMnode->pSdb;
+  sdbCancelFetch(pSdb, pIter);
+}

source/dnode/mnode/impl/src/mndProfile.c

@@ -21,6 +21,7 @@
 #include "mndStb.h"
 #include "mndUser.h"
 #include "tglobal.h"
+#include "version.h"
 
 #define QUERY_ID_SIZE 20
 #define QUERY_OBJ_ID_SIZE 18

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -19,6 +19,7 @@
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndMnode.h"
+#include "mndOffset.h"
 #include "mndShow.h"
 #include "mndStb.h"
 #include "mndTopic.h"
@@ -80,13 +81,13 @@ int32_t mndInitSubscribe(SMnode *pMnode) {
   return sdbSetTable(pMnode->pSdb, table);
 }
 
-static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *consumerGroup) {
+static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *cgroup) {
   SMqSubscribeObj *pSub = tNewSubscribeObj();
   if (pSub == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
-  char *key = mndMakeSubscribeKey(consumerGroup, pTopic->name);
+  char *key = mndMakeSubscribeKey(cgroup, pTopic->name);
   if (key == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     tDeleteSMqSubscribeObj(pSub);
@@ -289,9 +290,15 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) {
           strcpy(topicEp.topic, topicName);
           topicEp.vgs = taosArrayInit(vgsz, sizeof(SMqSubVgEp));
           for (int32_t k = 0; k < vgsz; k++) {
+            char           offsetKey[TSDB_PARTITION_KEY_LEN];
             SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, k);
-
-            SMqSubVgEp vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId};
+            SMqSubVgEp     vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId, .offset = -1};
+            mndMakePartitionKey(offsetKey, pConsumer->cgroup, topicName, pConsumerEp->vgId);
+            SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, offsetKey);
+            if (pOffsetObj != NULL) {
+              vgEp.offset = pOffsetObj->offset;
+              mndReleaseOffset(pMnode, pOffsetObj);
+            }
             taosArrayPush(topicEp.vgs, &vgEp);
           }
           taosArrayPush(rsp.topics, &topicEp);
@@ -870,7 +877,7 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) {
 
 SUB_ENCODE_OVER:
   tfree(buf);
-  if (terrno != 0) {
+  if (terrno != TSDB_CODE_SUCCESS) {
     mError("subscribe:%s, failed to encode to raw:%p since %s", pSub->key, pRaw, terrstr());
     sdbFreeRaw(pRaw);
     return NULL;
@@ -1085,6 +1092,8 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) {
         mDebug("create new subscription by consumer %ld, group: %s, topic %s", consumerId, cgroup, newTopicName);
         pSub = mndCreateSubscription(pMnode, pTopic, cgroup);
         createSub = true;
+
+        mndCreateOffset(pTrans, cgroup, newTopicName, pSub->unassignedVg);
       }
 
       SMqSubConsumer mqSubConsumer;

source/dnode/mnode/impl/src/mndTrans.c

@@ -71,7 +71,7 @@ int32_t mndInitTrans(SMnode *pMnode) {
                      .updateFp = (SdbUpdateFp)mndTransActionUpdate,
                      .deleteFp = (SdbDeleteFp)mndTransActionDelete};
 
-  mndSetMsgHandle(pMnode, TDMT_MND_TRANS, mndProcessTransReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransReq);
   mndSetMsgHandle(pMnode, TDMT_MND_KILL_TRANS, mndProcessKillTransReq);
 
   mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_TRANS, mndGetTransMeta);

source/dnode/mnode/impl/src/mnode.c

@@ -23,6 +23,7 @@
 #include "mndDnode.h"
 #include "mndFunc.h"
 #include "mndMnode.h"
+#include "mndOffset.h"
 #include "mndProfile.h"
 #include "mndQnode.h"
 #include "mndShow.h"
@@ -36,6 +37,10 @@
 #include "mndUser.h"
 #include "mndVgroup.h"
 
+#define MQ_TIMER_MS    3000
+#define TRNAS_TIMER_MS 6000
+#define TELEM_TIMER_MS 86400000
+
 int32_t mndSendReqToDnode(SMnode *pMnode, SEpSet *pEpSet, SRpcMsg *pMsg) {
   if (pMnode == NULL || pMnode->sendReqToDnodeFp == NULL) {
     terrno = TSDB_CODE_MND_NOT_READY;
@@ -73,46 +78,60 @@ static void *mndBuildTimerMsg(int32_t *pContLen) {
   return pReq;
 }
 
-static void mndTransReExecute(void *param, void *tmrId) {
+static void mndPullupTrans(void *param, void *tmrId) {
   SMnode *pMnode = param;
   if (mndIsMaster(pMnode)) {
     int32_t contLen = 0;
-    void *  pReq = mndBuildTimerMsg(&contLen);
-    SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS, .pCont = pReq, .contLen = contLen};
+    void   *pReq = mndBuildTimerMsg(&contLen);
+    SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS_TIMER, .pCont = pReq, .contLen = contLen};
     pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg);
   }
 
-  taosTmrReset(mndTransReExecute, 3000, pMnode, pMnode->timer, &pMnode->transTimer);
+  taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer);
 }
 
 static void mndCalMqRebalance(void *param, void *tmrId) {
   SMnode *pMnode = param;
   if (mndIsMaster(pMnode)) {
     int32_t contLen = 0;
-    void *  pReq = mndBuildTimerMsg(&contLen);
+    void   *pReq = mndBuildTimerMsg(&contLen);
     SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pReq, .contLen = contLen};
     pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg);
   }
 
-  taosTmrReset(mndCalMqRebalance, 3000, pMnode, pMnode->timer, &pMnode->mqTimer);
+  taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer);
+}
+
+static void mndPullupTelem(void *param, void *tmrId) {
+  SMnode *pMnode = param;
+  if (mndIsMaster(pMnode)) {
+    int32_t contLen = 0;
+    void   *pReq = mndBuildTimerMsg(&contLen);
+    SRpcMsg rpcMsg = {.msgType = TDMT_MND_TELEM_TIMER, .pCont = pReq, .contLen = contLen};
+    pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg);
+  }
+
+  taosTmrReset(mndPullupTelem, TELEM_TIMER_MS, pMnode, pMnode->timer, &pMnode->telemTimer);
 }
 
 static int32_t mndInitTimer(SMnode *pMnode) {
-  if (pMnode->timer == NULL) {
   pMnode->timer = taosTmrInit(5000, 200, 3600000, "MND");
+  if (pMnode->timer == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
   }
 
-  if (pMnode->timer == NULL) {
+  if (taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer)) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
 
-  if (taosTmrReset(mndTransReExecute, 6000, pMnode, pMnode->timer, &pMnode->transTimer)) {
+  if (taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer)) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
 
-  if (taosTmrReset(mndCalMqRebalance, 3000, pMnode, pMnode->timer, &pMnode->mqTimer)) {
+  if (taosTmrReset(mndPullupTelem, 60000, pMnode, pMnode->timer, &pMnode->telemTimer)) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
@@ -126,6 +145,8 @@ static void mndCleanupTimer(SMnode *pMnode) {
     pMnode->transTimer = NULL;
     taosTmrStop(pMnode->mqTimer);
     pMnode->mqTimer = NULL;
+    taosTmrStop(pMnode->telemTimer);
+    pMnode->telemTimer = NULL;
     taosTmrCleanUp(pMnode->timer);
     pMnode->timer = NULL;
   }
@@ -197,6 +218,7 @@ static int32_t mndInitSteps(SMnode *pMnode) {
   if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-consumer", mndInitConsumer, mndCleanupConsumer) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-subscribe", mndInitSubscribe, mndCleanupSubscribe) != 0) return -1;
+  if (mndAllocStep(pMnode, "mnode-offset", mndInitOffset, mndCleanupOffset) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1;
@@ -404,8 +426,8 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) {
     return NULL;
   }
 
-  if (pRpcMsg->msgType != TDMT_MND_TRANS && pRpcMsg->msgType != TDMT_MND_MQ_TIMER &&
-      pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE) {
+  if (pRpcMsg->msgType != TDMT_MND_TRANS_TIMER && pRpcMsg->msgType != TDMT_MND_MQ_TIMER &&
+      pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE && pRpcMsg->msgType != TDMT_MND_TELEM_TIMER) {
     SRpcConnInfo connInfo = {0};
     if ((pRpcMsg->msgType & 1U) && rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) {
       taosFreeQitem(pMsg);
@@ -440,7 +462,7 @@ void mndProcessMsg(SMnodeMsg *pMsg) {
   SMnode *pMnode = pMsg->pMnode;
   int32_t code = 0;
   tmsg_t  msgType = pMsg->rpcMsg.msgType;
-  void *  ahandle = pMsg->rpcMsg.ahandle;
+  void   *ahandle = pMsg->rpcMsg.ahandle;
   bool    isReq = (msgType & 1U);
 
   mTrace("msg:%p, type:%s will be processed, app:%p", pMsg, TMSG_INFO(msgType), ahandle);

source/dnode/vnode/src/vnd/vnodeWrite.c

@@ -22,14 +22,15 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
   for (int i = 0; i < taosArrayGetSize(pMsgs); i++) {
     pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i);
 
-    // ser request version
+    // set request version
     void   *pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
     int64_t ver = pVnode->state.processed++;
     taosEncodeFixedI64(&pBuf, ver);
 
     if (walWrite(pVnode->pWal, ver, pMsg->msgType, pMsg->pCont, pMsg->contLen) < 0) {
-      /*ASSERT(false);*/
       // TODO: handle error
+      /*ASSERT(false);*/
+      vError("vnode:%d  write wal error since %s", pVnode->vgId, terrstr());
     }
   }
 

source/libs/CMakeLists.txt

@@ -14,4 +14,5 @@ add_subdirectory(qcom)
 add_subdirectory(qworker)
 add_subdirectory(tfs)
 add_subdirectory(nodes)
+add_subdirectory(config)
 add_subdirectory(scalar)

source/libs/catalog/src/catalog.c

@@ -546,7 +546,7 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
 
   if (tbMeta->tableType != TSDB_CHILD_TABLE) {
     ctgReleaseDBCache(pCtg, dbCache);
-    ctgDebug("Got tbl from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname);
+    ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname);
     return TSDB_CODE_SUCCESS;
   }
   

source/libs/catalog/test/catalogTests.cpp

@@ -131,10 +131,8 @@ void ctgTestInitLogFile() {
 
   ctgDbgEnableDebug("api");
   
-  char temp[128] = {0};
-  sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix);
-  if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) {
-    printf("failed to open log file in directory:%s\n", tsLogDir);
+  if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
+    printf("failed to open log file in directory:%s\n", osLogDir());
   }
 }
 

source/libs/function/inc/builtins.h

@@ -46,6 +46,7 @@ typedef struct SBuiltinFuncDefinition {
   FExecGetEnv getEnvFunc;
   FExecInit initFunc;
   FExecProcess processFunc;
+  FScalarExecProcess sprocessFunc;
   FExecFinalize finalizeFunc;
 } SBuiltinFuncDefinition;