Merge remote-tracking branch 'origin/3.0' into feature/config

cmake/cmake.options

@@ -47,13 +47,13 @@ option(
 option(
     BUILD_WITH_UV
     "If build with libuv" 
-    OFF
+    ON  
 )
 
 option(
     BUILD_WITH_UV_TRANS
     "If build with libuv_trans " 
-    OFF 
+    ON 
 )
 
 option(

example/src/tmq.c

@@ -66,7 +66,7 @@ int32_t init_env() {
 }
 
 int32_t create_topic() {
-  printf("create topic");
+  printf("create topic\n");
   TAOS_RES* pRes;
   TAOS*     pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   if (pConn == NULL) {
@@ -91,6 +91,10 @@ int32_t create_topic() {
   return 0;
 }
 
+void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) {
+  printf("commit %d\n", resp);
+}
+
 tmq_t* build_consumer() {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   assert(pConn != NULL);
@@ -103,6 +107,7 @@ tmq_t* build_consumer() {
 
   tmq_conf_t* conf = tmq_conf_new();
   tmq_conf_set(conf, "group.id", "tg2");
+  tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print);
   tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0);
   return tmq;
 }
@@ -144,7 +149,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
 }
 
 void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
-  static const int MIN_COMMIT_COUNT = 1000;
+  static const int MIN_COMMIT_COUNT = 1;
 
   int            msg_count = 0;
   tmq_resp_err_t err;
@@ -214,6 +219,6 @@ int main(int argc, char* argv[]) {
   tmq_t*      tmq = build_consumer();
   tmq_list_t* topic_list = build_topic_list();
   /*perf_loop(tmq, topic_list);*/
-  basic_consume_loop(tmq, topic_list);
-  /*sync_consume_loop(tmq, topic_list);*/
+  /*basic_consume_loop(tmq, topic_list);*/
+  sync_consume_loop(tmq, topic_list);
 }

include/client/taos.h

@@ -198,8 +198,8 @@ DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLi
 /* --------------------------TMQ INTERFACE------------------------------- */
 
 enum tmq_resp_err_t {
+  TMQ_RESP_ERR__FAIL = -1,
   TMQ_RESP_ERR__SUCCESS = 0,
-  TMQ_RESP_ERR__FAIL = 1,
 };
 
 typedef enum tmq_resp_err_t tmq_resp_err_t;
@@ -226,7 +226,7 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
 DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list);
 #if 0
 DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq);
-DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_topic_vgroup_list_t** topics);
+DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics);
 #endif
 DLL_EXPORT tmq_message_t *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time);
 DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq);
@@ -238,6 +238,7 @@ DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *
 #if 0
 DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async);
 #endif
+DLL_EXPORT tmq_resp_err_t tmq_seek(tmq_t *tmq, const tmq_topic_vgroup_t *offset);
 /* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */
 
 enum tmq_conf_res_t {

include/common/common.h

@@ -16,7 +16,6 @@
 #ifndef TDENGINE_COMMON_H
 #define TDENGINE_COMMON_H
 
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -43,14 +42,16 @@ extern "C" {
 //   int16_t bytes;
 // } SSchema;
 
-#define TMQ_REQ_TYPE_COMMIT_ONLY 0
-#define TMQ_REQ_TYPE_CONSUME_ONLY 1
-#define TMQ_REQ_TYPE_CONSUME_AND_COMMIT 2
+enum {
+  TMQ_CONF__RESET_OFFSET__LATEST = -1,
+  TMQ_CONF__RESET_OFFSET__EARLIEAST = -2,
+  TMQ_CONF__RESET_OFFSET__NONE = -3,
+};
 
 typedef struct {
   uint32_t  numOfTables;
-  SArray   *pGroupList;
-  SHashObj *map;  // speedup acquire the tableQueryInfo by table uid
+  SArray*   pGroupList;
+  SHashObj* map;  // speedup acquire the tableQueryInfo by table uid
 } STableGroupInfo;
 
 typedef struct SColumnDataAgg {
@@ -79,14 +80,14 @@ typedef struct SConstantItem {
 
 // info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList);
 typedef struct SSDataBlock {
-  SColumnDataAgg *pBlockAgg;
-  SArray         *pDataBlock;    // SArray<SColumnInfoData>
-  SArray         *pConstantList; // SArray<SConstantItem>, it is a constant/tags value of the corresponding result value.
+  SColumnDataAgg* pBlockAgg;
+  SArray*         pDataBlock;  // SArray<SColumnInfoData>
+  SArray* pConstantList;       // SArray<SConstantItem>, it is a constant/tags value of the corresponding result value.
   SDataBlockInfo info;
 } SSDataBlock;
 
 typedef struct SVarColAttr {
-  int32_t *offset;    // start position for each entry in the list
+  int32_t* offset;    // start position for each entry in the list
   uint32_t length;    // used buffer size that contain the valid data
   uint32_t allocLen;  // allocated buffer size
 } SVarColAttr;
@@ -95,10 +96,10 @@ typedef struct SVarColAttr {
 // pBlockAgg->numOfNull == 0, no data are null.
 typedef struct SColumnInfoData {
   SColumnInfo info;     // TODO filter info needs to be removed
-  bool        hasNull;// if current column data has null value.
-  char       *pData;  // the corresponding block data in memory
+  bool        hasNull;  // if current column data has null value.
+  char*       pData;    // the corresponding block data in memory
   union {
-    char       *nullbitmap;  // bitmap, one bit for each item in the list
+    char*       nullbitmap;  // bitmap, one bit for each item in the list
     SVarColAttr varmeta;
   };
 } SColumnInfoData;
@@ -149,7 +150,6 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp
   int32_t tlen = 0;
   int32_t sz = 0;
   tlen += taosEncodeFixedI64(buf, pRsp->consumerId);
-  tlen += taosEncodeFixedI64(buf, pRsp->committedOffset);
   tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
   tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
   tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum);
@@ -170,7 +170,6 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp
 static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) {
   int32_t sz;
   buf = taosDecodeFixedI64(buf, &pRsp->consumerId);
-  buf = taosDecodeFixedI64(buf, &pRsp->committedOffset);
   buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
   buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
   buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);

include/common/tcompare.h

@@ -23,8 +23,12 @@
 extern "C" {
 #endif
 
-int32_t       compareStrPatternComp(const void* pLeft, const void* pRight);
-int32_t       compareWStrPatternComp(const void* pLeft, const void* pRight);
+int32_t compareStrPatternMatch(const void* pLeft, const void* pRight);
+int32_t compareStrPatternNotMatch(const void* pLeft, const void* pRight);
+
+int32_t compareWStrPatternMatch(const void* pLeft, const void* pRight);
+int32_t compareWStrPatternNotMatch(const void* pLeft, const void* pRight);
+
 __compar_fn_t getComparFunc(int32_t type, int32_t optr);
 __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
 int32_t       doCompare(const char* a, const char* b, int32_t type, size_t size);

include/common/tmsg.h

@@ -422,7 +422,7 @@ typedef struct {
 } SColumnInfo;
 
 typedef struct {
-  uint64_t uid;
+  int64_t uid;
   TSKEY   key;  // last accessed ts, for subscription
 } STableIdInfo;
 
@@ -555,7 +555,7 @@ int32_t tDeserializeSDropDbReq(void* buf, int32_t bufLen, SDropDbReq* pReq);
 
 typedef struct {
   char    db[TSDB_DB_FNAME_LEN];
-  uint64_t uid;
+  int64_t uid;
 } SDropDbRsp;
 
 int32_t tSerializeSDropDbRsp(void* buf, int32_t bufLen, SDropDbRsp* pRsp);
@@ -571,7 +571,7 @@ int32_t tDeserializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq);
 
 typedef struct {
   char    db[TSDB_DB_FNAME_LEN];
-  uint64_t uid;
+  int64_t uid;
   int32_t vgVersion;
   int32_t vgNum;
   int8_t  hashMethod;
@@ -725,7 +725,7 @@ typedef struct {
   int32_t  vgId;
   int32_t  dnodeId;
   char     db[TSDB_DB_FNAME_LEN];
-  uint64_t dbUid;
+  int64_t  dbUid;
   int32_t  vgVersion;
   int32_t  cacheBlockSize;
   int32_t  totalBlocks;
@@ -755,7 +755,7 @@ int32_t tDeserializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq*
 typedef struct {
   int32_t vgId;
   int32_t dnodeId;
-  uint64_t dbUid;
+  int64_t dbUid;
   char    db[TSDB_DB_FNAME_LEN];
 } SDropVnodeReq, SSyncVnodeReq, SCompactVnodeReq;
 
@@ -796,7 +796,7 @@ typedef struct {
   char     tbName[TSDB_TABLE_NAME_LEN];
   char     stbName[TSDB_TABLE_NAME_LEN];
   char     dbFName[TSDB_DB_FNAME_LEN];
-  uint64_t dbId;
+  int64_t  dbId;
   int32_t  numOfTags;
   int32_t  numOfColumns;
   int8_t   precision;
@@ -1268,7 +1268,7 @@ typedef struct {
 typedef struct {
   SMsgHead head;
   char     name[TSDB_TABLE_FNAME_LEN];
-  uint64_t tuid;
+  int64_t  tuid;
   int32_t  sverson;
   int32_t  execLen;
   char*    executor;
@@ -1279,11 +1279,11 @@ typedef struct {
 typedef struct {
   SMsgHead head;
   char     name[TSDB_TABLE_FNAME_LEN];
-  uint64_t tuid;
+  int64_t  tuid;
 } SDDropTopicReq;
 
 typedef struct SVCreateTbReq {
-  uint64_t ver;  // use a general definition
+  int64_t  ver;  // use a general definition
   char*    name;
   uint32_t ttl;
   uint32_t keep;
@@ -1314,7 +1314,7 @@ int32_t tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq);
 void*   tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq);
 
 typedef struct {
-  uint64_t ver;  // use a general definition
+  int64_t ver;  // use a general definition
   SArray* pArray;
 } SVCreateTbBatchReq;
 
@@ -1325,7 +1325,7 @@ int32_t tSerializeSVCreateTbBatchReq(void** buf, SVCreateTbBatchReq* pReq);
 void*   tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pReq);
 
 typedef struct {
-  uint64_t ver;
+  int64_t  ver;
   char*    name;
   uint8_t  type;
   tb_uid_t suid;
@@ -1760,35 +1760,19 @@ typedef struct {
   char    cgroup[TSDB_CONSUMER_GROUP_LEN];
 } SMqOffset;
 
-typedef struct {
-  int32_t vgId;
-  SArray* offsets;  // SArray<SMqOffset>
-} SMqVgOffsets;
-
 typedef struct {
   int32_t    num;
   SMqOffset* offsets;
-} SMqCMResetOffsetReq;
-
-typedef struct {
-  int32_t reserved;
-} SMqCMResetOffsetRsp;
-
-typedef struct {
-  int64_t      leftForVer;
-  SMqVgOffsets offsets;
-} SMqMVResetOffsetReq;
+} SMqCMCommitOffsetReq;
 
 typedef struct {
   int32_t reserved;
-} SMqMVResetOffsetRsp;
+} SMqCMCommitOffsetRsp;
 
 int32_t tEncodeSMqOffset(SCoder* encoder, const SMqOffset* pOffset);
 int32_t tDecodeSMqOffset(SCoder* decoder, SMqOffset* pOffset);
-int32_t tEncodeSMqCMResetOffsetReq(SCoder* encoder, const SMqCMResetOffsetReq* pReq);
-int32_t tDecodeSMqCMResetOffsetReq(SCoder* decoder, SMqCMResetOffsetReq* pReq);
-int32_t tEncodeSMqMVResetOffsetReq(SCoder* encoder, const SMqMVResetOffsetReq* pReq);
-int32_t tDecodeSMqMVResetOffsetReq(SCoder* decoder, SMqMVResetOffsetReq* pReq);
+int32_t tEncodeSMqCMCommitOffsetReq(SCoder* encoder, const SMqCMCommitOffsetReq* pReq);
+int32_t tDecodeSMqCMCommitOffsetReq(SCoder* decoder, SMqCMCommitOffsetReq* pReq);
 
 typedef struct {
   uint32_t nCols;
@@ -1870,7 +1854,6 @@ typedef struct {
 typedef struct {
   int64_t         consumerId;
   SSchemaWrapper* schemas;
-  int64_t         committedOffset;
   int64_t         reqOffset;
   int64_t         rspOffset;
   int32_t         skipLogNum;
@@ -1881,22 +1864,18 @@ typedef struct {
 // one req for one vg+topic
 typedef struct {
   SMsgHead head;
-  // 0: commit only, current offset
-  // 1: consume only, poll next offset
-  // 2: commit current and consume next offset
-  int32_t reqType;
 
-  int64_t reqId;
   int64_t consumerId;
   int64_t blockingTime;
   char    cgroup[TSDB_CONSUMER_GROUP_LEN];
 
-  int64_t offset;
+  int64_t currentOffset;
   char    topic[TSDB_TOPIC_FNAME_LEN];
 } SMqConsumeReq;
 
 typedef struct {
   int32_t vgId;
+  int64_t offset;
   SEpSet  epSet;
 } SMqSubVgEp;
 
@@ -1917,12 +1896,14 @@ static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taos
 static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) {
   int32_t tlen = 0;
   tlen += taosEncodeFixedI32(buf, pVgEp->vgId);
+  tlen += taosEncodeFixedI64(buf, pVgEp->offset);
   tlen += taosEncodeSEpSet(buf, &pVgEp->epSet);
   return tlen;
 }
 
 static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) {
   buf = taosDecodeFixedI32(buf, &pVgEp->vgId);
+  buf = taosDecodeFixedI64(buf, &pVgEp->offset);
   buf = taosDecodeSEpSet(buf, &pVgEp->epSet);
   return buf;
 }

include/common/tmsgdef.h

@@ -144,10 +144,10 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp)
-  TD_DEF_MSG_TYPE(TDMT_MND_RESET_OFFSET, "mnode-reset-offset", SMqCMResetOffsetReq, SMqCMResetOffsetRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq)
   TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg)
+  TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp)
 
   // Requests handled by VNODE
   TD_NEW_MSG_SEG(TDMT_VND_MSG)
@@ -195,4 +195,3 @@ enum {
   TDMT_MAX
 #endif
 };
-// clang-format on

include/common/tname.h

@@ -16,6 +16,11 @@
 #ifndef TDENGINE_TNAME_H
 #define TDENGINE_TNAME_H
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
 #include "tdef.h"
 #include "tmsg.h"
 
@@ -59,4 +64,9 @@ int32_t tNameSetAcctId(SName* dst, int32_t acctId);
 
 SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* name);
 
+#ifdef __cplusplus
+}
+#endif
+
+
 #endif  // TDENGINE_TNAME_H

include/common/ttypes.h

@@ -78,9 +78,12 @@ typedef struct {
       case TSDB_DATA_TYPE_UINT:                      \
         (_v) = (_finalType)GET_UINT32_VAL(_data);    \
         break;                                       \
-      default:                                       \
+      case TSDB_DATA_TYPE_INT:                       \
         (_v) = (_finalType)GET_INT32_VAL(_data);     \
         break;                                       \
+      default:                                       \
+        (_v) = (_finalType)varDataLen(_data);        \
+        break;                                       \
     }                                                \
   } while (0)
 
@@ -88,6 +91,8 @@ typedef struct {
   do {                                         \
     switch (_type) {                           \
       case TSDB_DATA_TYPE_BOOL:                \
+        *(bool *)(_v) = (bool)(_data);     \
+        break;                                 \
       case TSDB_DATA_TYPE_TINYINT:             \
         *(int8_t *)(_v) = (int8_t)(_data);     \
         break;                                 \
@@ -101,6 +106,7 @@ typedef struct {
         *(uint16_t *)(_v) = (uint16_t)(_data); \
         break;                                 \
       case TSDB_DATA_TYPE_BIGINT:              \
+      case TSDB_DATA_TYPE_TIMESTAMP:           \
         *(int64_t *)(_v) = (int64_t)(_data);   \
         break;                                 \
       case TSDB_DATA_TYPE_UBIGINT:             \
@@ -115,9 +121,11 @@ typedef struct {
       case TSDB_DATA_TYPE_UINT:                \
         *(uint32_t *)(_v) = (uint32_t)(_data); \
         break;                                 \
-      default:                                 \
+      case TSDB_DATA_TYPE_INT:                 \
         *(int32_t *)(_v) = (int32_t)(_data);   \
         break;                                 \
+      default:                                 \
+        break;                                 \
     }                                          \
   } while (0)
 
@@ -138,6 +146,9 @@ typedef struct {
 #define IS_VALID_FLOAT(_t)      ((_t) >= -FLT_MAX && (_t) <= FLT_MAX)
 #define IS_VALID_DOUBLE(_t)     ((_t) >= -DBL_MAX && (_t) <= DBL_MAX)
 
+#define IS_CONVERT_AS_SIGNED(_t) (IS_SIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP))
+#define IS_CONVERT_AS_UNSIGNED(_t) (IS_UNSIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL))
+
 static FORCE_INLINE bool isNull(const void *val, int32_t type) {
   switch (type) {
     case TSDB_DATA_TYPE_BOOL:
@@ -205,6 +216,7 @@ void* getDataMax(int32_t type);
 
 
 #define SET_DOUBLE_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_DOUBLE_NULL)
+#define SET_BIGINT_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_BIGINT_NULL)
 
 #ifdef __cplusplus
 }

include/dnode/mnode/sdb/sdb.h

@@ -113,14 +113,15 @@ typedef enum {
   SDB_USER = 7,
   SDB_AUTH = 8,
   SDB_ACCT = 9,
-  SDB_SUBSCRIBE = 10,
-  SDB_CONSUMER = 11,
-  SDB_TOPIC = 12,
-  SDB_VGROUP = 13,
-  SDB_STB = 14,
-  SDB_DB = 15,
-  SDB_FUNC = 16,
-  SDB_MAX = 17
+  SDB_OFFSET = 10,
+  SDB_SUBSCRIBE = 11,
+  SDB_CONSUMER = 12,
+  SDB_TOPIC = 13,
+  SDB_VGROUP = 14,
+  SDB_STB = 15,
+  SDB_DB = 16,
+  SDB_FUNC = 17,
+  SDB_MAX = 18
 } ESdbType;
 
 typedef struct SSdb SSdb;

include/libs/function/function.h

@@ -226,13 +226,19 @@ typedef struct SAggFunctionInfo {
   int32_t (*dataReqFunc)(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId);
 } SAggFunctionInfo;
 
-struct SScalarFuncParam;
+typedef struct SScalarParam {
+  void*   data;
+  bool    colData;
+  int32_t num;
+  int32_t type;
+  int32_t bytes;
+} SScalarParam;
 
 typedef struct SScalarFunctionInfo {
   char      name[FUNCTIONS_NAME_MAX_LENGTH];
   int8_t    type;              // scalar function or aggregation function
   uint32_t  functionId;        // index of scalar function
-  void     (*process)(struct SScalarFuncParam* pOutput, size_t numOfInput, const struct SScalarFuncParam *pInput);
+  void     (*process)(struct SScalarParam* pOutput, size_t numOfInput, const struct SScalarParam *pInput);
 } SScalarFunctionInfo;
 
 typedef struct SMultiFunctionsDesc {
@@ -285,10 +291,6 @@ int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num);
 bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry);
 bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry);
 
-struct SScalarFunctionSupport* createScalarFuncSupport(int32_t num);
-void destroyScalarFuncSupport(struct SScalarFunctionSupport* pSupport, int32_t num);
-struct SScalarFunctionSupport* getScalarFuncSupport(struct SScalarFunctionSupport* pSupport, int32_t index);
-
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 // fill api
 struct SFillInfo;

include/libs/function/functionMgt.h

@@ -21,6 +21,7 @@ extern "C" {
 #endif
 
 #include "querynodes.h"
+#include "function.h"
 
 typedef enum EFunctionType {
   // aggregate function
@@ -117,6 +118,13 @@ typedef struct SFuncExecFuncs {
   FExecFinalize finalize;
 } SFuncExecFuncs;
 
+typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+
+typedef struct SScalarFuncExecFuncs {
+  FScalarExecProcess process;
+} SScalarFuncExecFuncs;
+
+
 int32_t fmFuncMgtInit();
 
 int32_t fmGetFuncInfo(const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType);
@@ -134,6 +142,7 @@ bool fmIsTimeorderFunc(int32_t funcId);
 int32_t fmFuncScanType(int32_t funcId);
 
 int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet);
+int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet);
 
 #ifdef __cplusplus
 }

include/libs/nodes/nodes.h

@@ -61,22 +61,27 @@ typedef enum ENodeType {
   QUERY_NODE_INTERVAL_WINDOW,
   QUERY_NODE_NODE_LIST,
   QUERY_NODE_FILL,
+  QUERY_NODE_RAW_EXPR, // Only be used in parser module.
   QUERY_NODE_COLUMN_REF,
   QUERY_NODE_TARGET,
-
-  // Only be used in parser module.
-  QUERY_NODE_RAW_EXPR,
+  QUERY_NODE_TUPLE_DESC,
+  QUERY_NODE_SLOT_DESC,
 
   // Statement nodes are used in parser and planner module.
   QUERY_NODE_SET_OPERATOR,
   QUERY_NODE_SELECT_STMT,
   QUERY_NODE_SHOW_STMT,
 
+  // logic plan node
   QUERY_NODE_LOGIC_PLAN_SCAN,
   QUERY_NODE_LOGIC_PLAN_JOIN,
-  QUERY_NODE_LOGIC_PLAN_FILTER,
   QUERY_NODE_LOGIC_PLAN_AGG,
-  QUERY_NODE_LOGIC_PLAN_PROJECT
+  QUERY_NODE_LOGIC_PLAN_PROJECT,
+
+  // physical plan node
+  QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN,
+  QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN,
+  QUERY_NODE_PHYSICAL_PLAN_PROJECT
 } ENodeType;
 
 /**
@@ -132,7 +137,7 @@ bool nodesEqualNode(const SNode* a, const SNode* b);
 SNode* nodesCloneNode(const SNode* pNode);
 SNodeList* nodesCloneList(const SNodeList* pList);
 
-int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen);
+int32_t nodesNodeToString(const SNode* pNode, bool format, char** pStr, int32_t* pLen);
 int32_t nodesStringToNode(const char* pStr, SNode** pNode);
 
 #ifdef __cplusplus

include/libs/nodes/plannodes.h

@@ -21,6 +21,7 @@ extern "C" {
 #endif
 
 #include "querynodes.h"
+#include "tmsg.h"
 
 typedef struct SLogicNode {
   ENodeType type;
@@ -31,10 +32,20 @@ typedef struct SLogicNode {
   struct SLogicNode* pParent;
 } SLogicNode;
 
+typedef enum EScanType {
+  SCAN_TYPE_TAG,
+  SCAN_TYPE_TABLE,
+  SCAN_TYPE_STABLE,
+  SCAN_TYPE_STREAM
+} EScanType;
+
 typedef struct SScanLogicNode {
   SLogicNode node;
   SNodeList* pScanCols;
   struct STableMeta* pMeta;
+  EScanType scanType;
+  uint8_t scanFlag;         // denotes reversed scan of data or not
+  STimeWindow scanRange;
 } SScanLogicNode;
 
 typedef struct SJoinLogicNode {
@@ -43,10 +54,6 @@ typedef struct SJoinLogicNode {
   SNode* pOnConditions;
 } SJoinLogicNode;
 
-typedef struct SFilterLogicNode {
-  SLogicNode node;
-} SFilterLogicNode;
-
 typedef struct SAggLogicNode {
   SLogicNode node;
   SNodeList* pGroupKeys;
@@ -58,6 +65,56 @@ typedef struct SProjectLogicNode {
   SNodeList* pProjections;
 } SProjectLogicNode;
 
+typedef struct SSlotDescNode {
+  ENodeType type;
+  int16_t slotId;
+  SDataType dataType;
+  int16_t srcTupleId;
+  int16_t srcSlotId;
+  bool reserve;
+  bool output;
+} SSlotDescNode;
+
+typedef struct STupleDescNode {
+  ENodeType type;
+  int16_t tupleId;
+  SNodeList* pSlots;
+} STupleDescNode;
+
+typedef struct SPhysiNode {
+  ENodeType type;
+  STupleDescNode outputTuple;
+  SNode* pConditions;
+  SNodeList* pChildren;
+  struct SPhysiNode* pParent;
+} SPhysiNode;
+
+typedef struct SScanPhysiNode {
+  SPhysiNode  node;
+  SNodeList* pScanCols;
+  uint64_t uid;           // unique id of the table
+  int8_t tableType;
+  int32_t order;         // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC
+  int32_t count;         // repeat count
+  int32_t reverse;       // reverse scan count
+} SScanPhysiNode;
+
+typedef SScanPhysiNode SSystemTableScanPhysiNode;
+typedef SScanPhysiNode STagScanPhysiNode;
+
+typedef struct STableScanPhysiNode {
+  SScanPhysiNode scan;
+  uint8_t scanFlag;         // denotes reversed scan of data or not
+  STimeWindow scanRange;
+} STableScanPhysiNode;
+
+typedef STableScanPhysiNode STableSeqScanPhysiNode;
+
+typedef struct SProjectPhysiNode {
+  SPhysiNode node;
+  SNodeList* pProjections;
+} SProjectPhysiNode;
+
 #ifdef __cplusplus
 }
 #endif

include/libs/nodes/querynodes.h

@@ -68,6 +68,13 @@ typedef struct SColumnRefNode {
   int16_t columnId;
 } SColumnRefNode;
 
+typedef struct STargetNode {
+  ENodeType type;
+  int16_t tupleId;
+  int16_t slotId;
+  SNode* pExpr;
+} STargetNode;
+
 typedef struct SValueNode {
   SExprNode node; // QUERY_NODE_VALUE
   char* literal;
@@ -81,45 +88,6 @@ typedef struct SValueNode {
   } datum;
 } SValueNode;
 
-typedef enum EOperatorType {
-  // arithmetic operator
-  OP_TYPE_ADD = 1,
-  OP_TYPE_SUB,
-  OP_TYPE_MULTI,
-  OP_TYPE_DIV,
-  OP_TYPE_MOD,
-
-  // bit operator
-  OP_TYPE_BIT_AND,
-  OP_TYPE_BIT_OR,
-
-  // comparison operator
-  OP_TYPE_GREATER_THAN,
-  OP_TYPE_GREATER_EQUAL,
-  OP_TYPE_LOWER_THAN,
-  OP_TYPE_LOWER_EQUAL,
-  OP_TYPE_EQUAL,
-  OP_TYPE_NOT_EQUAL,
-  OP_TYPE_IN,
-  OP_TYPE_NOT_IN,
-  OP_TYPE_LIKE,
-  OP_TYPE_NOT_LIKE,
-  OP_TYPE_MATCH,
-  OP_TYPE_NMATCH,
-  OP_TYPE_IS_NULL,
-  OP_TYPE_IS_NOT_NULL,
-  OP_TYPE_IS_TRUE,
-  OP_TYPE_IS_FALSE,
-  OP_TYPE_IS_UNKNOWN,
-  OP_TYPE_IS_NOT_TRUE,
-  OP_TYPE_IS_NOT_FALSE,
-  OP_TYPE_IS_NOT_UNKNOWN,
-
-  // json operator
-  OP_TYPE_JSON_GET_VALUE,
-  OP_TYPE_JSON_CONTAINS
-} EOperatorType;
-
 typedef struct SOperatorNode {
   SExprNode node; // QUERY_NODE_OPERATOR
   EOperatorType opType;
@@ -127,11 +95,6 @@ typedef struct SOperatorNode {
   SNode* pRight;
 } SOperatorNode;
 
-typedef enum ELogicConditionType {
-  LOGIC_COND_TYPE_AND,
-  LOGIC_COND_TYPE_OR,
-  LOGIC_COND_TYPE_NOT,
-} ELogicConditionType;
 
 typedef struct SLogicConditionNode {
   SExprNode node; // QUERY_NODE_LOGIC_CONDITION
@@ -141,6 +104,7 @@ typedef struct SLogicConditionNode {
 
 typedef struct SNodeListNode {
   ENodeType type; // QUERY_NODE_NODE_LIST
+  SDataType dataType;
   SNodeList* pNodeList;
 } SNodeListNode;
 
@@ -307,6 +271,8 @@ bool nodesIsJsonOp(const SOperatorNode* pOp);
 bool nodesIsTimeorderQuery(const SNode* pQuery);
 bool nodesIsTimelineQuery(const SNode* pQuery);
 
+void* nodesGetValueFromNode(SValueNode *pNode);
+
 #ifdef __cplusplus
 }
 #endif

include/libs/scalar/filter.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef TDENGINE_FILTER_H
+#define TDENGINE_FILTER_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SFilterInfo SFilterInfo;
+typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **);
+
+
+enum {
+  FLT_OPTION_NO_REWRITE = 1,
+  FLT_OPTION_TIMESTAMP = 2,
+  FLT_OPTION_NEED_UNIQE = 4,
+};
+
+typedef struct SFilterColumnParam{
+  int32_t numOfCols;
+  SArray* pDataBlock;
+} SFilterColumnParam;
+
+
+extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t options);
+extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols);
+extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param, filer_get_col_from_id fp);
+extern int32_t filterSetDataFromColId(SFilterInfo *info, void *param, filer_get_col_from_id fp);
+extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win);
+extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar);
+extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo);
+extern void filterFreeInfo(SFilterInfo *info);
+extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_FILTER_H

include/libs/scalar/scalar.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef TDENGINE_SCALAR_H
+#define TDENGINE_SCALAR_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "function.h"
+#include "nodes.h"
+#include "querynodes.h"
+
+typedef struct SFilterInfo SFilterInfo;
+
+
+int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes);
+int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst);
+int32_t scalarGetOperatorParamNum(EOperatorType type);
+int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type);
+
+int32_t vectorGetConvertType(int32_t type1, int32_t type2);
+int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_SCALAR_H

include/libs/sync/sync.h

@@ -137,22 +137,21 @@ typedef struct {
 
 } SSyncInfo;
 
-// will be defined in syncInt.h, here just for complie
-typedef struct SSyncNode {
-} SSyncNode;
+struct SSyncNode;
+typedef struct SSyncNode SSyncNode;
 
 int32_t syncInit();
 void    syncCleanUp();
 
-int64_t syncStart(const SSyncInfo*);
+int64_t syncStart(const SSyncInfo* pSyncInfo);
 void    syncStop(int64_t rid);
-int32_t syncReconfig(int64_t rid, const SSyncCfg*);
+int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
 
 // int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak);
 int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak);
 
 ESyncState syncGetMyRole(int64_t rid);
-void       syncGetNodesRole(int64_t rid, SNodesRole*);
+void       syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole);
 
 extern int32_t sDebugFlag;
 

include/libs/transport/trpc.h

@@ -64,6 +64,7 @@ typedef struct SRpcInit {
   int8_t   connType;      // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS
   int      idleTime;      // milliseconds, 0 means idle timer is disabled
 
+  bool noPool;  //  create conn pool or not
   // the following is for client app ecurity only
   char *user;     // user name
   char  spi;      // security parameter index

include/util/compare.h

@@ -49,10 +49,19 @@ int32_t WCSPatternMatch(const wchar_t *pattern, const wchar_t *str, size_t size,
 
 int32_t taosArrayCompareString(const void *a, const void *b);
 
-int32_t setCompareBytes1(const void *pLeft, const void *pRight);
-int32_t setCompareBytes2(const void *pLeft, const void *pRight);
-int32_t setCompareBytes4(const void *pLeft, const void *pRight);
-int32_t setCompareBytes8(const void *pLeft, const void *pRight);
+int32_t setChkInBytes1(const void *pLeft, const void *pRight);
+int32_t setChkInBytes2(const void *pLeft, const void *pRight);
+int32_t setChkInBytes4(const void *pLeft, const void *pRight);
+int32_t setChkInBytes8(const void *pLeft, const void *pRight);
+
+int32_t setChkNotInBytes1(const void *pLeft, const void *pRight);
+int32_t setChkNotInBytes2(const void *pLeft, const void *pRight);
+int32_t setChkNotInBytes4(const void *pLeft, const void *pRight);
+int32_t setChkNotInBytes8(const void *pLeft, const void *pRight);
+
+int32_t compareChkInString(const void *pLeft, const void *pRight);
+int32_t compareChkNotInString(const void *pLeft, const void *pRight);
+
 
 int32_t compareInt8Val(const void *pLeft, const void *pRight);
 int32_t compareInt16Val(const void *pLeft, const void *pRight);
@@ -74,7 +83,6 @@ int32_t compareStrRegexComp(const void *pLeft, const void *pRight);
 int32_t compareStrRegexCompMatch(const void *pLeft, const void *pRight);
 int32_t compareStrRegexCompNMatch(const void *pLeft, const void *pRight);
 
-int32_t compareFindItemInSet(const void *pLeft, const void *pRight);
 
 int32_t compareInt8ValDesc(const void *pLeft, const void *pRight);
 int32_t compareInt16ValDesc(const void *pLeft, const void *pRight);
@@ -92,6 +100,9 @@ int32_t compareUint64ValDesc(const void *pLeft, const void *pRight);
 int32_t compareLenPrefixedStrDesc(const void *pLeft, const void *pRight);
 int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight);
 
+__compar_fn_t getComparFunc(int32_t type, int32_t optr);
+
+
 #ifdef __cplusplus
 }
 #endif

include/util/taoserror.h

@@ -20,6 +20,8 @@
 extern "C" {
 #endif
 
+// clang-format off
+
 #define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code))))
  
 #define TAOS_SYSTEM_ERROR(code)             (0x80ff0000 | (code))
@@ -262,6 +264,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_CONSUMER_NOT_EXIST        TAOS_DEF_ERROR_CODE(0, 0x03E7)
 #define TSDB_CODE_MND_UNSUPPORTED_TOPIC         TAOS_DEF_ERROR_CODE(0, 0x03E8)
 #define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E9)
+#define TSDB_CODE_MND_OFFSET_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03EA)
 #define TSDB_CODE_MND_MQ_PLACEHOLDER            TAOS_DEF_ERROR_CODE(0, 0x03F0)
 
 // dnode

include/util/tdef.h

@@ -13,6 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+// clang-format off
+
 #ifndef _TD_UTIL_DEF_H
 #define _TD_UTIL_DEF_H
 
@@ -108,33 +110,52 @@ do { \
   (src) = (void *)((char *)src + sizeof(type));\
 } while(0)
 
+typedef enum EOperatorType {
+  // arithmetic operator
+  OP_TYPE_ADD = 1,
+  OP_TYPE_SUB,
+  OP_TYPE_MULTI,
+  OP_TYPE_DIV,
+  OP_TYPE_MOD,
+
+  // bit operator
+  OP_TYPE_BIT_AND,
+  OP_TYPE_BIT_OR,
+
+  // comparison operator
+  OP_TYPE_GREATER_THAN,
+  OP_TYPE_GREATER_EQUAL,
+  OP_TYPE_LOWER_THAN,
+  OP_TYPE_LOWER_EQUAL,
+  OP_TYPE_EQUAL,
+  OP_TYPE_NOT_EQUAL,
+  OP_TYPE_IN,
+  OP_TYPE_NOT_IN,
+  OP_TYPE_LIKE,
+  OP_TYPE_NOT_LIKE,
+  OP_TYPE_MATCH,
+  OP_TYPE_NMATCH,
+  OP_TYPE_IS_NULL,
+  OP_TYPE_IS_NOT_NULL,
+  OP_TYPE_IS_TRUE,
+  OP_TYPE_IS_FALSE,
+  OP_TYPE_IS_UNKNOWN,
+  OP_TYPE_IS_NOT_TRUE,
+  OP_TYPE_IS_NOT_FALSE,
+  OP_TYPE_IS_NOT_UNKNOWN,
+
+  // json operator
+  OP_TYPE_JSON_GET_VALUE,
+  OP_TYPE_JSON_CONTAINS
+} EOperatorType;
+
+
+typedef enum ELogicConditionType {
+  LOGIC_COND_TYPE_AND,
+  LOGIC_COND_TYPE_OR,
+  LOGIC_COND_TYPE_NOT,
+} ELogicConditionType;
 
-// TODO: check if below is necessary
-#define TSDB_RELATION_INVALID     0
-#define TSDB_RELATION_LESS        1
-#define TSDB_RELATION_GREATER     2
-#define TSDB_RELATION_EQUAL       3
-#define TSDB_RELATION_LESS_EQUAL  4
-#define TSDB_RELATION_GREATER_EQUAL 5
-#define TSDB_RELATION_NOT_EQUAL   6
-#define TSDB_RELATION_LIKE        7
-#define TSDB_RELATION_ISNULL      8
-#define TSDB_RELATION_NOTNULL     9
-#define TSDB_RELATION_IN          10
-
-#define TSDB_RELATION_AND         11
-#define TSDB_RELATION_OR          12
-#define TSDB_RELATION_NOT         13
-
-#define TSDB_RELATION_MATCH       14
-#define TSDB_RELATION_NMATCH      15
-
-#define TSDB_BINARY_OP_ADD        4000
-#define TSDB_BINARY_OP_SUBTRACT   4001
-#define TSDB_BINARY_OP_MULTIPLY   4002
-#define TSDB_BINARY_OP_DIVIDE     4003
-#define TSDB_BINARY_OP_REMAINDER  4004
-#define TSDB_BINARY_OP_CONCAT     4005
 
 #define FUNCTION_CEIL        4500
 #define FUNCTION_FLOOR       4501
@@ -146,9 +167,6 @@ do { \
 #define FUNCTION_LTRIM       4802
 #define FUNCTION_RTRIM       4803
 
-#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN))
-#define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER))
-
 #define TSDB_NAME_DELIMITER_LEN   1
 
 #define TSDB_UNI_LEN              24
@@ -180,6 +198,7 @@ do { \
 #define TSDB_TOPIC_FNAME_LEN      TSDB_TABLE_FNAME_LEN
 #define TSDB_CONSUMER_GROUP_LEN   192
 #define TSDB_SUBSCRIBE_KEY_LEN    (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2)
+#define TSDB_PARTITION_KEY_LEN    (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2)
 #define TSDB_COL_NAME_LEN         65
 #define TSDB_MAX_SAVED_SQL_LEN    TSDB_MAX_COLUMNS * 64
 #define TSDB_MAX_SQL_LEN          TSDB_PAYLOAD_SIZE
@@ -333,9 +352,6 @@ do { \
 #define TSDB_QUERY_TYPE_NON_TYPE        0x00u     // none type
 #define TSDB_QUERY_TYPE_FREE_RESOURCE   0x01u     // free qhandle at vnode
 
-#define TSDB_QUERY_TYPE_NON_TYPE      0x00u  // none type
-#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u  // free qhandle at vnode
-
 #define TSDB_META_COMPACT_RATIO         0       // disable tsdb meta compact by default
 
 

include/util/thash.h

@@ -55,6 +55,8 @@ uint32_t taosIntHash_64(const char *key, uint32_t len);
 
 _hash_fn_t taosGetDefaultHashFunction(int32_t type);
 
+_equal_fn_t taosGetDefaultEqualFunction(int32_t type);
+
 typedef struct SHashNode {
   struct SHashNode *next;
   uint32_t          hashVal;     // the hash value of key
@@ -258,6 +260,8 @@ void* taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen);
  */
 void taosHashRelease(SHashObj *pHashObj, void *p);
 
+void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp);
+
 
 #ifdef __cplusplus
 }

include/util/tjson.h

@@ -30,6 +30,7 @@ void tjsonDelete(SJson* pJson);
 SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName);
 
 int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t number);
+int32_t tjsonAddDoubleToObject(SJson* pJson, const char* pName, const double number);
 int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal);
 int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem);
 int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem);
@@ -42,6 +43,7 @@ int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj);
 typedef int32_t (*FFromJson)(const SJson* pJson, void* pObj);
 
 char* tjsonToString(const SJson* pJson);
+char* tjsonToUnformattedString(const SJson* pJson);
 
 #ifdef __cplusplus
 }

include/util/types.h

@@ -66,6 +66,7 @@ static FORCE_INLINE double taos_align_get_double(const char *pBuf) {
 // #else
   #define GET_FLOAT_VAL(x)       (*(float *)(x))
   #define GET_DOUBLE_VAL(x)      (*(double *)(x))
+  #define SET_BIGINT_VAL(x, y)  { (*(int64_t *)(x))  = (int64_t)(y);       }
   #define SET_FLOAT_VAL(x, y)  { (*(float *)(x))  = (float)(y);       }
   #define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y);      }
   #define SET_FLOAT_PTR(x, y)  { (*(float *)(x))  = (*(float *)(y));  }

source/client/src/tmq.c

@@ -47,20 +47,22 @@ struct tmq_topic_vgroup_list_t {
 struct tmq_conf_t {
   char           clientId[256];
   char           groupId[256];
-  bool auto_commit;
+  int8_t         auto_commit;
+  int8_t         resetOffset;
+  tmq_commit_cb* commit_cb;
   /*char*          ip;*/
   /*uint16_t       port;*/
-  tmq_commit_cb* commit_cb;
 };
 
 struct tmq_t {
   // conf
   char           groupId[256];
   char           clientId[256];
-  bool           autoCommit;
+  int8_t         autoCommit;
   SRWLatch       lock;
   int64_t        consumerId;
   int32_t        epoch;
+  int32_t        resetOffsetCfg;
   int64_t        status;
   tsem_t         rspSem;
   STscObj*       pTscObj;
@@ -79,7 +81,6 @@ typedef struct {
   // statistics
   int64_t pollCnt;
   // offset
-  int64_t committedOffset;
   int64_t currentOffset;
   // connection info
   int32_t vgId;
@@ -116,20 +117,16 @@ typedef struct {
 
 typedef struct {
   tmq_t* tmq;
-  SMqClientVg* pVg;
+  /*SMqClientVg* pVg;*/
   int32_t        async;
   tsem_t         rspSem;
-} SMqCommitCbParam;
-
-typedef struct {
-  tmq_t*         tmq;
-  tsem_t         rspSem;
   tmq_resp_err_t rspErr;
-} SMqResetOffsetParam;
+} SMqCommitCbParam;
 
 tmq_conf_t* tmq_conf_new() {
   tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t));
   conf->auto_commit = false;
+  conf->resetOffset = TMQ_CONF__RESET_OFFSET__LATEST;
   return conf;
 }
 
@@ -157,6 +154,20 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
       return TMQ_CONF_INVALID;
     }
   }
+  if (strcmp(key, "auto.offset.reset") == 0) {
+    if (strcmp(value, "none") == 0) {
+      conf->resetOffset = TMQ_CONF__RESET_OFFSET__NONE;
+      return TMQ_CONF_OK;
+    } else if (strcmp(value, "earliest") == 0) {
+      conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST;
+      return TMQ_CONF_OK;
+    } else if (strcmp(value, "latest") == 0) {
+      conf->resetOffset = TMQ_CONF__RESET_OFFSET__LATEST;
+      return TMQ_CONF_OK;
+    } else {
+      return TMQ_CONF_INVALID;
+    }
+  }
   return TMQ_CONF_UNKNOWN;
 }
 
@@ -190,14 +201,12 @@ int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) {
   if (pParam->tmq->commit_cb) {
     pParam->tmq->commit_cb(pParam->tmq, rspErr, NULL, NULL);
   }
-  if (!pParam->async) tsem_post(&pParam->rspSem);
-  return 0;
-}
-
-int32_t tmqResetOffsetCb(void* param, const SDataBuf* pMsg, int32_t code) {
-  SMqResetOffsetParam* pParam = (SMqResetOffsetParam*)param;
-  pParam->rspErr = code;
+  if (!pParam->async)
     tsem_post(&pParam->rspSem);
+  else {
+    tsem_destroy(&pParam->rspSem);
+    free(param);
+  }
   return 0;
 }
 
@@ -216,6 +225,7 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
   strcpy(pTmq->groupId, conf->groupId);
   pTmq->autoCommit = conf->auto_commit;
   pTmq->commit_cb = conf->commit_cb;
+  pTmq->resetOffsetCfg = conf->resetOffset;
 
   tsem_init(&pTmq->rspSem, 0, 0);
   pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1);
@@ -223,18 +233,40 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
   return pTmq;
 }
 
-tmq_resp_err_t tmq_reset_offset(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets) {
+tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) {
+  // TODO: add read write lock
   SRequestObj*   pRequest = NULL;
+  tmq_resp_err_t resp = TMQ_RESP_ERR__SUCCESS;
   // build msg
   // send to mnode
-  SMqCMResetOffsetReq req;
+  SMqCMCommitOffsetReq req;
+  SArray*              pArray = NULL;
+
+  if (offsets == NULL) {
+    pArray = taosArrayInit(0, sizeof(SMqOffset));
+    for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
+      SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
+      for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
+        SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
+        SMqOffset    offset;
+        strcpy(offset.topicName, pTopic->topicName);
+        strcpy(offset.cgroup, tmq->groupId);
+        offset.vgId = pVg->vgId;
+        offset.offset = pVg->currentOffset;
+        taosArrayPush(pArray, &offset);
+      }
+    }
+    req.num = pArray->size;
+    req.offsets = pArray->pData;
+  } else {
     req.num = offsets->cnt;
     req.offsets = (SMqOffset*)offsets->elems;
+  }
 
   SCoder encoder;
 
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER);
-  tEncodeSMqCMResetOffsetReq(&encoder, &req);
+  tEncodeSMqCMCommitOffsetReq(&encoder, &req);
   int32_t tlen = encoder.pos;
   void*   buf = malloc(tlen);
   if (buf == NULL) {
@@ -244,32 +276,41 @@ tmq_resp_err_t tmq_reset_offset(tmq_t* tmq, const tmq_topic_vgroup_list_t* offse
   tCoderClear(&encoder);
 
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, tlen, TD_ENCODER);
-  tEncodeSMqCMResetOffsetReq(&encoder, &req);
+  tEncodeSMqCMCommitOffsetReq(&encoder, &req);
   tCoderClear(&encoder);
 
-  pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_RESET_OFFSET);
+  pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_MQ_COMMIT_OFFSET);
   if (pRequest == NULL) {
     tscError("failed to malloc request");
   }
 
-  SMqResetOffsetParam param = {0};
-  tsem_init(&param.rspSem, 0, 0);
-  param.tmq = tmq;
+  SMqCommitCbParam* pParam = malloc(sizeof(SMqCommitCbParam));
+  if (pParam == NULL) {
+    return -1;
+  }
+  pParam->tmq = tmq;
+  tsem_init(&pParam->rspSem, 0, 0);
 
   pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen};
 
   SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
-  sendInfo->param = &param;
-  sendInfo->fp = tmqResetOffsetCb;
+  sendInfo->param = pParam;
+  sendInfo->fp = tmqCommitCb;
   SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
 
   int64_t transporterId = 0;
   asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
 
-  tsem_wait(&param.rspSem);
-  tsem_destroy(&param.rspSem);
+  if (!async) {
+    tsem_wait(&pParam->rspSem);
+    resp = pParam->rspErr;
+  }
 
-  return param.rspErr;
+  if (pArray) {
+    taosArrayDestroy(pArray);
+  }
+
+  return resp;
 }
 
 tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
@@ -641,8 +682,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
         // clang-format off
         SMqClientVg clientVg = {
             .pollCnt = 0,
-            .committedOffset = -1,
-            .currentOffset = -1,
+            .currentOffset = pVgEp->offset,
             .vgId = pVgEp->vgId,
             .epSet = pVgEp->epSet
         };
@@ -708,23 +748,51 @@ END:
   return 0;
 }
 
-SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, int32_t type, SMqClientTopic* pTopic,
-                                      SMqClientVg* pVg) {
+tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) {
+  const SMqOffset* pOffset = &offset->offset;
+  if (strcmp(pOffset->cgroup, tmq->groupId) != 0) {
+    return TMQ_RESP_ERR__FAIL;
+  }
+  int32_t sz = taosArrayGetSize(tmq->clientTopics);
+  for (int32_t i = 0; i < sz; i++) {
+    SMqClientTopic* clientTopic = taosArrayGet(tmq->clientTopics, i);
+    if (strcmp(clientTopic->topicName, pOffset->topicName) == 0) {
+      int32_t vgSz = taosArrayGetSize(clientTopic->vgs);
+      for (int32_t j = 0; j < vgSz; j++) {
+        SMqClientVg* pVg = taosArrayGet(clientTopic->vgs, j);
+        if (pVg->vgId == pOffset->vgId) {
+          pVg->currentOffset = pOffset->offset;
+          return TMQ_RESP_ERR__SUCCESS;
+        }
+      }
+    }
+  }
+  return TMQ_RESP_ERR__FAIL;
+}
+
+SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, SMqClientTopic* pTopic, SMqClientVg* pVg) {
+  int64_t reqOffset;
+  if (pVg->currentOffset >= 0) {
+    reqOffset = pVg->currentOffset;
+  } else {
+    if (tmq->resetOffsetCfg == TMQ_CONF__RESET_OFFSET__NONE) {
+      tscError("unable to poll since no committed offset but reset offset is set to none");
+      return NULL;
+    }
+    reqOffset = tmq->resetOffsetCfg;
+  }
+
   SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq));
   if (pReq == NULL) {
     return NULL;
   }
-  pReq->reqType = type;
+
   strcpy(pReq->topic, pTopic->topicName);
-  pReq->blockingTime = blocking_time;
-  pReq->consumerId = tmq->consumerId;
   strcpy(pReq->cgroup, tmq->groupId);
 
-  if (type == TMQ_REQ_TYPE_COMMIT_ONLY) {
-    pReq->offset = pVg->currentOffset;
-  } else {
-    pReq->offset = pVg->currentOffset + 1;
-  }
+  pReq->blockingTime = blocking_time;
+  pReq->consumerId = tmq->consumerId;
+  pReq->currentOffset = reqOffset;
 
   pReq->head.vgId = htonl(pVg->vgId);
   pReq->head.contLen = htonl(sizeof(SMqConsumeReq));
@@ -743,13 +811,13 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
 
   if (taosArrayGetSize(tmq->clientTopics) == 0) {
     tscDebug("consumer:%ld poll but not assigned", tmq->consumerId);
-    printf("over1\n");
+    /*printf("over1\n");*/
     usleep(blocking_time * 1000);
     return NULL;
   }
   SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx);
   if (taosArrayGetSize(pTopic->vgs) == 0) {
-    printf("over2\n");
+    /*printf("over2\n");*/
     usleep(blocking_time * 1000);
     return NULL;
   }
@@ -760,8 +828,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
     pTopic->nextVgIdx = (pTopic->nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs);
     SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx);
     /*printf("consume vg %d, offset %ld\n", pVg->vgId, pVg->currentOffset);*/
-    int32_t        reqType = tmq->autoCommit ? TMQ_REQ_TYPE_CONSUME_AND_COMMIT : TMQ_REQ_TYPE_CONSUME_ONLY;
-    SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, reqType, pTopic, pVg);
+    SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, pTopic, pVg);
     if (pReq == NULL) {
       ASSERT(false);
       usleep(blocking_time * 1000);
@@ -821,6 +888,7 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
   /*return pRequest;*/
 }
 
+#if 0
 tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) {
   if (tmq_topic_vgroup_list != NULL) {
     // TODO
@@ -831,7 +899,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v
     SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
     for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
       SMqClientVg*   pVg = taosArrayGet(pTopic->vgs, j);
-      SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, TMQ_REQ_TYPE_COMMIT_ONLY, pTopic, pVg);
+      SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, pTopic, pVg);
 
       SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME);
       pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)};
@@ -858,6 +926,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v
 
   return 0;
 }
+#endif
 
 void tmq_message_destroy(tmq_message_t* tmq_message) {
   if (tmq_message == NULL) return;

source/common/src/tcompare.c

@@ -15,116 +15,6 @@
 
 #include "tcompare.h"
 
-int32_t compareStrPatternComp(const void* pLeft, const void* pRight) {
-  SPatternCompareInfo pInfo = {'%', '_'};
-
-  assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN);
-  char *pattern = calloc(varDataLen(pRight) + 1, sizeof(char));
-  memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
-
-  size_t sz = varDataLen(pLeft);
-  char *buf = malloc(sz + 1);
-  memcpy(buf, varDataVal(pLeft), sz);
-  buf[sz] = 0;
-
-  int32_t ret = patternMatch(pattern, buf, sz, &pInfo);
-  free(buf);
-  free(pattern);
-  return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
-}
-
-int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
-  SPatternCompareInfo pInfo = {'%', '_'};
-
-  assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE);
-
-  wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t));
-  memcpy(pattern, varDataVal(pRight), varDataLen(pRight));
-
-  int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft)/TSDB_NCHAR_SIZE, &pInfo);
-  free(pattern);
-
-  return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
-}
-
-__compar_fn_t getComparFunc(int32_t type, int32_t optr) {
-  __compar_fn_t comparFn = NULL;
-
-  if (optr == TSDB_RELATION_IN && (type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR)) {
-    switch (type) {
-      case TSDB_DATA_TYPE_BOOL:
-      case TSDB_DATA_TYPE_TINYINT:
-      case TSDB_DATA_TYPE_UTINYINT:
-        return setCompareBytes1;
-      case TSDB_DATA_TYPE_SMALLINT:
-      case TSDB_DATA_TYPE_USMALLINT:
-        return setCompareBytes2;
-      case TSDB_DATA_TYPE_INT:
-      case TSDB_DATA_TYPE_UINT:
-      case TSDB_DATA_TYPE_FLOAT:
-        return setCompareBytes4;
-      case TSDB_DATA_TYPE_BIGINT:
-      case TSDB_DATA_TYPE_UBIGINT:
-      case TSDB_DATA_TYPE_DOUBLE:
-      case TSDB_DATA_TYPE_TIMESTAMP:
-        return setCompareBytes8;
-      default:
-        assert(0);
-    }
-  }
-
-  switch (type) {
-    case TSDB_DATA_TYPE_BOOL:
-    case TSDB_DATA_TYPE_TINYINT:   comparFn = compareInt8Val;   break;
-    case TSDB_DATA_TYPE_SMALLINT:  comparFn = compareInt16Val;  break;
-    case TSDB_DATA_TYPE_INT:       comparFn = compareInt32Val;  break;
-    case TSDB_DATA_TYPE_BIGINT:
-    case TSDB_DATA_TYPE_TIMESTAMP: comparFn = compareInt64Val;  break;
-    case TSDB_DATA_TYPE_FLOAT:     comparFn = compareFloatVal;  break;
-    case TSDB_DATA_TYPE_DOUBLE:    comparFn = compareDoubleVal; break;
-    case TSDB_DATA_TYPE_BINARY: {
-      if (optr == TSDB_RELATION_MATCH) {
-        comparFn = compareStrRegexCompMatch;
-      } else if (optr == TSDB_RELATION_NMATCH) {
-        comparFn = compareStrRegexCompNMatch;
-      } else if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */
-        comparFn = compareStrPatternComp;
-      } else if (optr == TSDB_RELATION_IN) {
-        comparFn = compareFindItemInSet;
-      } else { /* normal relational comparFn */
-        comparFn = compareLenPrefixedStr;
-      }
-
-      break;
-    }
-
-    case TSDB_DATA_TYPE_NCHAR: {
-      if (optr == TSDB_RELATION_MATCH) {
-        comparFn = compareStrRegexCompMatch;
-      } else if (optr == TSDB_RELATION_NMATCH) {
-        comparFn = compareStrRegexCompNMatch;
-      } else if (optr == TSDB_RELATION_LIKE) {
-        comparFn = compareWStrPatternComp;
-      } else if (optr == TSDB_RELATION_IN) {
-        comparFn = compareFindItemInSet;
-      } else {
-        comparFn = compareLenPrefixedWStr;
-      }
-      break;
-    }
-
-    case TSDB_DATA_TYPE_UTINYINT:  comparFn = compareUint8Val; break;
-    case TSDB_DATA_TYPE_USMALLINT: comparFn = compareUint16Val;break;
-    case TSDB_DATA_TYPE_UINT:      comparFn = compareUint32Val;break;
-    case TSDB_DATA_TYPE_UBIGINT:   comparFn = compareUint64Val;break;
-
-    default:
-      comparFn = compareInt32Val;
-      break;
-  }
-
-  return comparFn;
-}
 
 __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) {
   __compar_fn_t comparFn = NULL;

source/common/src/tmsg.c

@@ -285,7 +285,7 @@ int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchR
 int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
   int32_t tlen = 0;
 
-  tlen += taosEncodeFixedU64(buf, pReq->ver);
+  tlen += taosEncodeFixedI64(buf, pReq->ver);
   tlen += taosEncodeString(buf, pReq->name);
   tlen += taosEncodeFixedU32(buf, pReq->ttl);
   tlen += taosEncodeFixedU32(buf, pReq->keep);
@@ -330,7 +330,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
 }
 
 void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
-  buf = taosDecodeFixedU64(buf, &(pReq->ver));
+  buf = taosDecodeFixedI64(buf, &(pReq->ver));
   buf = taosDecodeString(buf, &(pReq->name));
   buf = taosDecodeFixedU32(buf, &(pReq->ttl));
   buf = taosDecodeFixedU32(buf, &(pReq->keep));
@@ -380,7 +380,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
 int32_t tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) {
   int32_t tlen = 0;
 
-  tlen += taosEncodeFixedU64(buf, pReq->ver);
+  tlen += taosEncodeFixedI64(buf, pReq->ver);
   tlen += taosEncodeFixedU32(buf, taosArrayGetSize(pReq->pArray));
   for (size_t i = 0; i < taosArrayGetSize(pReq->pArray); i++) {
     SVCreateTbReq *pCreateTbReq = taosArrayGet(pReq->pArray, i);
@@ -393,7 +393,7 @@ int32_t tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) {
 void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) {
   uint32_t nsize = 0;
 
-  buf = taosDecodeFixedU64(buf, &pReq->ver);
+  buf = taosDecodeFixedI64(buf, &pReq->ver);
   buf = taosDecodeFixedU32(buf, &nsize);
   pReq->pArray = taosArrayInit(nsize, sizeof(SVCreateTbReq));
   for (size_t i = 0; i < nsize; i++) {
@@ -407,14 +407,14 @@ void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) {
 
 int32_t tSerializeSVDropTbReq(void **buf, SVDropTbReq *pReq) {
   int32_t tlen = 0;
-  tlen += taosEncodeFixedU64(buf, pReq->ver);
+  tlen += taosEncodeFixedI64(buf, pReq->ver);
   tlen += taosEncodeString(buf, pReq->name);
   tlen += taosEncodeFixedU8(buf, pReq->type);
   return tlen;
 }
 
 void *tDeserializeSVDropTbReq(void *buf, SVDropTbReq *pReq) {
-  buf = taosDecodeFixedU64(buf, &pReq->ver);
+  buf = taosDecodeFixedI64(buf, &pReq->ver);
   buf = taosDecodeString(buf, &pReq->name);
   buf = taosDecodeFixedU8(buf, &pReq->type);
   return buf;
@@ -1393,7 +1393,7 @@ int32_t tSerializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) {
 
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeCStr(&encoder, pRsp->db) < 0) return -1;
-  if (tEncodeU64(&encoder, pRsp->uid) < 0) return -1;
+  if (tEncodeI64(&encoder, pRsp->uid) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -1407,7 +1407,7 @@ int32_t tDeserializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) {
 
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pRsp->db) < 0) return -1;
-  if (tDecodeU64(&decoder, &pRsp->uid) < 0) return -1;
+  if (tDecodeI64(&decoder, &pRsp->uid) < 0) return -1;
   tEndDecode(&decoder);
 
   tCoderClear(&decoder);
@@ -1468,7 +1468,7 @@ int32_t tDeserializeSSyncDbReq(void *buf, int32_t bufLen, SSyncDbReq *pReq) {
 
 static int32_t tSerializeSUseDbRspImp(SCoder *pEncoder, SUseDbRsp *pRsp) {
   if (tEncodeCStr(pEncoder, pRsp->db) < 0) return -1;
-  if (tEncodeU64(pEncoder, pRsp->uid) < 0) return -1;
+  if (tEncodeI64(pEncoder, pRsp->uid) < 0) return -1;
   if (tEncodeI32(pEncoder, pRsp->vgVersion) < 0) return -1;
   if (tEncodeI32(pEncoder, pRsp->vgNum) < 0) return -1;
   if (tEncodeI8(pEncoder, pRsp->hashMethod) < 0) return -1;
@@ -1518,7 +1518,7 @@ int32_t tSerializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pRsp
 
 int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) {
   if (tDecodeCStrTo(pDecoder, pRsp->db) < 0) return -1;
-  if (tDecodeU64(pDecoder, &pRsp->uid) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pRsp->uid) < 0) return -1;
   if (tDecodeI32(pDecoder, &pRsp->vgVersion) < 0) return -1;
   if (tDecodeI32(pDecoder, &pRsp->vgNum) < 0) return -1;
   if (tDecodeI8(pDecoder, &pRsp->hashMethod) < 0) return -1;
@@ -1661,7 +1661,7 @@ static int32_t tEncodeSTableMetaRsp(SCoder *pEncoder, STableMetaRsp *pRsp) {
   if (tEncodeCStr(pEncoder, pRsp->tbName) < 0) return -1;
   if (tEncodeCStr(pEncoder, pRsp->stbName) < 0) return -1;
   if (tEncodeCStr(pEncoder, pRsp->dbFName) < 0) return -1;
-  if (tEncodeU64(pEncoder, pRsp->dbId) < 0) return -1;
+  if (tEncodeI64(pEncoder, pRsp->dbId) < 0) return -1;
   if (tEncodeI32(pEncoder, pRsp->numOfTags) < 0) return -1;
   if (tEncodeI32(pEncoder, pRsp->numOfColumns) < 0) return -1;
   if (tEncodeI8(pEncoder, pRsp->precision) < 0) return -1;
@@ -1684,7 +1684,7 @@ static int32_t tDecodeSTableMetaRsp(SCoder *pDecoder, STableMetaRsp *pRsp) {
   if (tDecodeCStrTo(pDecoder, pRsp->tbName) < 0) return -1;
   if (tDecodeCStrTo(pDecoder, pRsp->stbName) < 0) return -1;
   if (tDecodeCStrTo(pDecoder, pRsp->dbFName) < 0) return -1;
-  if (tDecodeU64(pDecoder, &pRsp->dbId) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pRsp->dbId) < 0) return -1;
   if (tDecodeI32(pDecoder, &pRsp->numOfTags) < 0) return -1;
   if (tDecodeI32(pDecoder, &pRsp->numOfColumns) < 0) return -1;
   if (tDecodeI8(pDecoder, &pRsp->precision) < 0) return -1;
@@ -2093,7 +2093,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR
   if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
-  if (tEncodeU64(&encoder, pReq->dbUid) < 0) return -1;
+  if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->cacheBlockSize) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1;
@@ -2133,7 +2133,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
   if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
-  if (tDecodeU64(&decoder, &pReq->dbUid) < 0) return -1;
+  if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->cacheBlockSize) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1;
@@ -2171,7 +2171,7 @@ int32_t tSerializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq)
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
-  if (tEncodeU64(&encoder, pReq->dbUid) < 0) return -1;
+  if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
   tEndEncode(&encoder);
 
@@ -2187,7 +2187,7 @@ int32_t tDeserializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
-  if (tDecodeU64(&decoder, &pReq->dbUid) < 0) return -1;
+  if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
   tEndDecode(&decoder);
 
@@ -2356,35 +2356,7 @@ int32_t tDecodeSMqOffset(SCoder *decoder, SMqOffset *pOffset) {
   return 0;
 }
 
-int32_t tEncodeSMqVgOffsets(SCoder *encoder, const SMqVgOffsets *pOffsets) {
-  if (tStartEncode(encoder) < 0) return -1;
-  if (tEncodeI32(encoder, pOffsets->vgId) < 0) return -1;
-  int32_t sz = taosArrayGetSize(pOffsets->offsets);
-  if (tEncodeI32(encoder, sz) < 0) return -1;
-  for (int32_t i = 0; i < sz; i++) {
-    SMqOffset *offset = taosArrayGet(pOffsets->offsets, i);
-    if (tEncodeSMqOffset(encoder, offset) < 0) return -1;
-  }
-  tEndEncode(encoder);
-  return encoder->pos;
-}
-
-int32_t tDecodeSMqVgOffsets(SCoder *decoder, SMqVgOffsets *pOffsets) {
-  int32_t sz;
-  if (tStartDecode(decoder) < 0) return -1;
-  if (tDecodeI32(decoder, &pOffsets->vgId) < 0) return -1;
-  if (tDecodeI32(decoder, &sz) < 0) return -1;
-  pOffsets->offsets = taosArrayInit(sz, sizeof(SMqOffset));
-  for (int32_t i = 0; i < sz; i++) {
-    SMqOffset offset;
-    if (tDecodeSMqOffset(decoder, &offset) < 0) return -1;
-    taosArrayPush(pOffsets->offsets, &offset);
-  }
-  tEndDecode(decoder);
-  return 0;
-}
-
-int32_t tEncodeSMqCMResetOffsetReq(SCoder *encoder, const SMqCMResetOffsetReq *pReq) {
+int32_t tEncodeSMqCMCommitOffsetReq(SCoder *encoder, const SMqCMCommitOffsetReq *pReq) {
   if (tStartEncode(encoder) < 0) return -1;
   if (tEncodeI32(encoder, pReq->num) < 0) return -1;
   for (int32_t i = 0; i < pReq->num; i++) {
@@ -2394,7 +2366,7 @@ int32_t tEncodeSMqCMResetOffsetReq(SCoder *encoder, const SMqCMResetOffsetReq *p
   return encoder->pos;
 }
 
-int32_t tDecodeSMqCMResetOffsetReq(SCoder *decoder, SMqCMResetOffsetReq *pReq) {
+int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq) {
   if (tStartDecode(decoder) < 0) return -1;
   if (tDecodeI32(decoder, &pReq->num) < 0) return -1;
   pReq->offsets = TCODER_MALLOC(pReq->num * sizeof(SMqOffset), decoder);
@@ -2405,23 +2377,3 @@ int32_t tDecodeSMqCMResetOffsetReq(SCoder *decoder, SMqCMResetOffsetReq *pReq) {
   tEndDecode(decoder);
   return 0;
 }
-
-#if 0
-int32_t tEncodeSMqMVResetOffsetReq(SCoder *encoder, const SMqMVResetOffsetReq *pReq) {
-  if (tEncodeI64(encoder, pReq->leftForVer) < 0) return -1;
-  for (int32_t i = 0; i < pReq->num; i++) {
-    tEncodeSMqOffset(encoder, &pReq->offsets[i]);
-  }
-  return encoder->pos;
-}
-
-int32_t tDecodeSMqMVResetOffsetReq(SCoder *decoder, SMqMVResetOffsetReq *pReq) {
-  if (tDecodeI32(decoder, &pReq->num) < 0) return -1;
-  pReq->offsets = TCODER_MALLOC(pReq->num * sizeof(SMqOffset), decoder);
-  if (pReq->offsets == NULL) return -1;
-  for (int32_t i = 0; i < pReq->num; i++) {
-    tDecodeSMqOffset(decoder, &pReq->offsets[i]);
-  }
-  return 0;
-}
-#endif

source/common/src/tname.c

@@ -31,7 +31,7 @@ SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFil
   }
 
   assert(src->filterstr == 0 || src->filterstr == 1);
-  assert(!(src->lowerRelOptr == TSDB_RELATION_INVALID && src->upperRelOptr == TSDB_RELATION_INVALID));
+  assert(!(src->lowerRelOptr == 0 && src->upperRelOptr == 0));
 
   return pFilter;
 }

source/common/src/ttypes.c

@@ -585,7 +585,7 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) {
 }
 
 void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) {
-  if (optr == TSDB_BINARY_OP_ADD) {
+  if (optr == OP_TYPE_ADD) {
     switch (type) {
       case TSDB_DATA_TYPE_TINYINT:
         *((int8_t *)dst) = GET_INT8_VAL(s1) + GET_INT8_VAL(s2);

source/dnode/mgmt/impl/src/dndTransport.c

@@ -113,6 +113,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
   pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_TOPIC)] = dndProcessMnodeWriteMsg;
   pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_TOPIC)] = dndProcessMnodeWriteMsg;
   pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SUBSCRIBE)] = dndProcessMnodeWriteMsg;
+  pMgmt->msgFp[TMSG_INDEX(TDMT_MND_MQ_COMMIT_OFFSET)] = dndProcessMnodeWriteMsg;
   /*pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBSCRIBE_RSP)] = dndProcessMnodeWriteMsg;*/
   pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN_RSP)] = dndProcessMnodeWriteMsg;
   pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_REB_RSP)] = dndProcessMnodeWriteMsg;
@@ -193,6 +194,7 @@ static int32_t dndInitClient(SDnode *pDnode) {
   rpcInit.ckey = INTERNAL_CKEY;
   rpcInit.spi = 1;
   rpcInit.parent = pDnode;
+  rpcInit.noPool = true;
 
   char pass[TSDB_PASSWORD_LEN + 1] = {0};
   taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass);

source/dnode/mgmt/impl/test/sut/inc/client.h

@@ -21,13 +21,18 @@ class TestClient {
   bool Init(const char* user, const char* pass, const char* fqdn, uint16_t port);
   void Cleanup();
 
+  void DoInit();
+
   SRpcMsg* SendReq(SRpcMsg* pReq);
   void     SetRpcRsp(SRpcMsg* pRsp);
   tsem_t*  GetSem();
+  void     Restart();
 
  private:
   char     fqdn[TSDB_FQDN_LEN];
   uint16_t port;
+  char     user[128];
+  char     pass[128];
   void*    clientRpc;
   SRpcMsg* pRsp;
   tsem_t   sem;

source/dnode/mgmt/impl/test/sut/inc/sut.h

@@ -23,6 +23,7 @@
 #include "tmsg.h"
 #include "tdataformat.h"
 #include "tglobal.h"
+#include "tmsg.h"
 #include "tnote.h"
 #include "trpc.h"
 #include "tthread.h"
@@ -38,6 +39,7 @@ class Testbase {
   void     Restart();
   void     ServerStop();
   void     ServerStart();
+  void     ClientRestart();
   SRpcMsg* SendReq(tmsg_t msgType, void* pCont, int32_t contLen);
 
  private:

source/dnode/mgmt/impl/test/sut/src/client.cpp

@@ -13,33 +13,41 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tep.h"
 #include "sut.h"
+#include "tep.h"
 
 static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) {
   TestClient* client = (TestClient*)parent;
   client->SetRpcRsp(pRsp);
-  uInfo("response:%s from dnode, code:0x%x", TMSG_INFO(pRsp->msgType), pRsp->code);
+  uInfo("x response:%s from dnode, code:0x%x, msgSize: %d", TMSG_INFO(pRsp->msgType), pRsp->code, pRsp->contLen);
   tsem_post(client->GetSem());
 }
 
-void TestClient::SetRpcRsp(SRpcMsg* pRsp) { this->pRsp = pRsp; };
+void TestClient::SetRpcRsp(SRpcMsg* rsp) {
+  if (this->pRsp) {
+    free(this->pRsp);
+  }
+  this->pRsp = (SRpcMsg*)calloc(1, sizeof(SRpcMsg));
+  this->pRsp->msgType = rsp->msgType;
+  this->pRsp->code = rsp->code;
+  this->pRsp->pCont = rsp->pCont;
+  this->pRsp->contLen = rsp->contLen;
+};
 
 tsem_t* TestClient::GetSem() { return &sem; }
 
-bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint16_t port) {
+void TestClient::DoInit() {
   char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0};
   taosEncryptPass_c((uint8_t*)pass, strlen(pass), secretEncrypt);
-
   SRpcInit rpcInit;
   memset(&rpcInit, 0, sizeof(rpcInit));
-  rpcInit.label = (char*)"DND-C";
+  rpcInit.label = (char*)"shell";
   rpcInit.numOfThreads = 1;
   rpcInit.cfp = processClientRsp;
   rpcInit.sessions = 1024;
   rpcInit.connType = TAOS_CONN_CLIENT;
   rpcInit.idleTime = 30 * 1000;
-  rpcInit.user = (char*)user;
+  rpcInit.user = (char*)this->user;
   rpcInit.ckey = (char*)"key";
   rpcInit.parent = this;
   rpcInit.secret = (char*)secretEncrypt;
@@ -47,11 +55,16 @@ bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint
 
   clientRpc = rpcOpen(&rpcInit);
   ASSERT(clientRpc);
+  tsem_init(&this->sem, 0, 0);
+}
 
-  tsem_init(&sem, 0, 0);
+bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint16_t port) {
   strcpy(this->fqdn, fqdn);
+  strcpy(this->user, user);
+  strcpy(this->pass, pass);
   this->port = port;
-
+  this->pRsp = NULL;
+  this->DoInit();
   return true;
 }
 
@@ -60,11 +73,16 @@ void TestClient::Cleanup() {
   rpcClose(clientRpc);
 }
 
+void TestClient::Restart() {
+  this->Cleanup();
+  this->DoInit();
+}
 SRpcMsg* TestClient::SendReq(SRpcMsg* pReq) {
   SEpSet epSet = {0};
   addEpIntoEpSet(&epSet, fqdn, port);
   rpcSendRequest(clientRpc, &epSet, pReq, NULL);
   tsem_wait(&sem);
+  uInfo("y response:%s from dnode, code:0x%x, msgSize: %d", TMSG_INFO(pRsp->msgType), pRsp->code, pRsp->contLen);
 
   return pRsp;
 }

source/dnode/mgmt/impl/test/sut/src/sut.cpp

@@ -21,9 +21,9 @@ void Testbase::InitLog(const char* path) {
   mDebugFlag = 143;
   cDebugFlag = 0;
   jniDebugFlag = 0;
-  tmrDebugFlag = 0;
-  uDebugFlag = 0;
-  rpcDebugFlag = 0;
+  tmrDebugFlag = 143;
+  uDebugFlag = 143;
+  rpcDebugFlag = 143;
   qDebugFlag = 0;
   wDebugFlag = 0;
   sDebugFlag = 0;
@@ -62,16 +62,21 @@ void Testbase::Init(const char* path, int16_t port) {
 
 void Testbase::Cleanup() {
   tFreeSTableMetaRsp(&metaRsp);
-  server.Stop();
   client.Cleanup();
+  taosMsleep(10);
+  server.Stop();
   dndCleanup();
 }
 
-void Testbase::Restart() { server.Restart(); }
+void Testbase::Restart() {
+  server.Restart();
+  client.Restart();
+}
 
 void Testbase::ServerStop() { server.Stop(); }
 
 void Testbase::ServerStart() { server.DoStart(); }
+void Testbase::ClientRestart() { client.Restart(); }
 
 SRpcMsg* Testbase::SendReq(tmsg_t msgType, void* pCont, int32_t contLen) {
   SRpcMsg rpcMsg = {0};

source/dnode/mnode/impl/inc/mndDef.h

@@ -123,6 +123,7 @@ typedef enum {
   TRN_TYPE_DROP_TOPIC = 1015,
   TRN_TYPE_SUBSCRIBE = 1016,
   TRN_TYPE_REBALANCE = 1017,
+  TRN_TYPE_COMMIT_OFFSET = 1018,
   TRN_TYPE_BASIC_SCOPE_END,
   TRN_TYPE_GLOBAL_SCOPE = 2000,
   TRN_TYPE_CREATE_DNODE = 2001,
@@ -176,7 +177,7 @@ typedef struct {
   SArray*    undoActions;
   int64_t    createdTime;
   int64_t    lastExecTime;
-  uint64_t   dbUid;
+  int64_t    dbUid;
   char       dbname[TSDB_DB_FNAME_LEN];
   char       lastError[TSDB_TRANS_ERROR_LEN];
 } STrans;
@@ -309,7 +310,7 @@ typedef struct {
   char    createUser[TSDB_USER_LEN];
   int64_t createdTime;
   int64_t updateTime;
-  uint64_t uid;
+  int64_t uid;
   int32_t cfgVersion;
   int32_t vgVersion;
   int8_t  hashMethod;  // default is 1
@@ -346,8 +347,8 @@ typedef struct {
   char     db[TSDB_DB_FNAME_LEN];
   int64_t  createdTime;
   int64_t  updateTime;
-  uint64_t uid;
-  uint64_t dbUid;
+  int64_t  uid;
+  int64_t  dbUid;
   int32_t  version;
   int32_t  nextColId;
   int32_t  numOfColumns;
@@ -465,6 +466,24 @@ static FORCE_INLINE void tDeleteSMqSubConsumer(SMqSubConsumer* pSubConsumer) {
   }
 }
 
+typedef struct {
+  char    key[TSDB_PARTITION_KEY_LEN];
+  int64_t offset;
+} SMqOffsetObj;
+
+static FORCE_INLINE int32_t tEncodeSMqOffsetObj(void** buf, const SMqOffsetObj* pOffset) {
+  int32_t tlen = 0;
+  tlen += taosEncodeString(buf, pOffset->key);
+  tlen += taosEncodeFixedI64(buf, pOffset->offset);
+  return tlen;
+}
+
+static FORCE_INLINE void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset) {
+  buf = taosDecodeStringTo(buf, pOffset->key);
+  buf = taosDecodeFixedI64(buf, &pOffset->offset);
+  return buf;
+}
+
 typedef struct {
   char    key[TSDB_SUBSCRIBE_KEY_LEN];
   int32_t status;

source/dnode/mnode/impl/inc/mndOffset.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_MND_OFFSET_H_
+#define _TD_MND_OFFSET_H_
+
+#include "mndInt.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t mndInitOffset(SMnode *pMnode);
+void    mndCleanupOffset(SMnode *pMnode);
+
+SMqOffsetObj *mndAcquireOffset(SMnode *pMnode, const char *key);
+void          mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset);
+
+SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset);
+SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw);
+
+int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs);
+
+static FORCE_INLINE int32_t mndMakePartitionKey(char *key, const char *cgroup, const char *topicName, int32_t vgId) {
+  return snprintf(key, TSDB_PARTITION_KEY_LEN, "%d:%s:%s", vgId, cgroup, topicName);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_MND_OFFSET_H_*/

source/dnode/mnode/impl/src/mndOffset.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "mndOffset.h"
+#include "mndAuth.h"
+#include "mndDb.h"
+#include "mndDnode.h"
+#include "mndMnode.h"
+#include "mndShow.h"
+#include "mndStb.h"
+#include "mndTrans.h"
+#include "mndUser.h"
+#include "mndVgroup.h"
+#include "tname.h"
+
+#define MND_OFFSET_VER_NUMBER   1
+#define MND_OFFSET_RESERVE_SIZE 64
+
+static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset);
+static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset);
+static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOffset, SMqOffsetObj *pNewOffset);
+static int32_t mndProcessCommitOffsetReq(SMnodeMsg *pReq);
+
+int32_t mndInitOffset(SMnode *pMnode) {
+  SSdbTable table = {.sdbType = SDB_OFFSET,
+                     .keyType = SDB_KEY_BINARY,
+                     .encodeFp = (SdbEncodeFp)mndOffsetActionEncode,
+                     .decodeFp = (SdbDecodeFp)mndOffsetActionDecode,
+                     .insertFp = (SdbInsertFp)mndOffsetActionInsert,
+                     .updateFp = (SdbUpdateFp)mndOffsetActionUpdate,
+                     .deleteFp = (SdbDeleteFp)mndOffsetActionDelete};
+
+  mndSetMsgHandle(pMnode, TDMT_MND_MQ_COMMIT_OFFSET, mndProcessCommitOffsetReq);
+
+  return sdbSetTable(pMnode->pSdb, table);
+}
+
+void mndCleanupOffset(SMnode *pMnode) {}
+
+SSdbRaw *mndOffsetActionEncode(SMqOffsetObj *pOffset) {
+  terrno = TSDB_CODE_OUT_OF_MEMORY;
+  void   *buf = NULL;
+  int32_t tlen = tEncodeSMqOffsetObj(NULL, pOffset);
+  int32_t size = sizeof(int32_t) + tlen + MND_OFFSET_RESERVE_SIZE;
+
+  SSdbRaw *pRaw = sdbAllocRaw(SDB_OFFSET, MND_OFFSET_VER_NUMBER, size);
+  if (pRaw == NULL) goto OFFSET_ENCODE_OVER;
+
+  buf = malloc(tlen);
+  if (buf == NULL) goto OFFSET_ENCODE_OVER;
+
+  void *abuf = buf;
+  tEncodeSMqOffsetObj(&abuf, pOffset);
+
+  int32_t dataPos = 0;
+  SDB_SET_INT32(pRaw, dataPos, tlen, OFFSET_ENCODE_OVER);
+  SDB_SET_BINARY(pRaw, dataPos, buf, tlen, OFFSET_ENCODE_OVER);
+  SDB_SET_RESERVE(pRaw, dataPos, MND_OFFSET_RESERVE_SIZE, OFFSET_ENCODE_OVER);
+  SDB_SET_DATALEN(pRaw, dataPos, OFFSET_ENCODE_OVER);
+
+  terrno = TSDB_CODE_SUCCESS;
+
+OFFSET_ENCODE_OVER:
+  tfree(buf);
+  if (terrno != TSDB_CODE_SUCCESS) {
+    mError("offset:%s, failed to encode to raw:%p since %s", pOffset->key, pRaw, terrstr());
+    sdbFreeRaw(pRaw);
+    return NULL;
+  }
+
+  mTrace("offset:%s, encode to raw:%p, row:%p", pOffset->key, pRaw, pOffset);
+  return pRaw;
+}
+
+SSdbRow *mndOffsetActionDecode(SSdbRaw *pRaw) {
+  terrno = TSDB_CODE_OUT_OF_MEMORY;
+  void *buf = NULL;
+
+  int8_t sver = 0;
+  if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto OFFSET_DECODE_OVER;
+
+  if (sver != MND_OFFSET_VER_NUMBER) {
+    terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
+    goto OFFSET_DECODE_OVER;
+  }
+
+  int32_t  size = sizeof(SMqOffsetObj);
+  SSdbRow *pRow = sdbAllocRow(size);
+  if (pRow == NULL) goto OFFSET_DECODE_OVER;
+
+  SMqOffsetObj *pOffset = sdbGetRowObj(pRow);
+  if (pOffset == NULL) goto OFFSET_DECODE_OVER;
+
+  int32_t dataPos = 0;
+  int32_t tlen;
+  SDB_GET_INT32(pRaw, dataPos, &tlen, OFFSET_DECODE_OVER);
+  buf = malloc(tlen + 1);
+  if (buf == NULL) goto OFFSET_DECODE_OVER;
+  SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OFFSET_DECODE_OVER);
+  SDB_GET_RESERVE(pRaw, dataPos, MND_OFFSET_RESERVE_SIZE, OFFSET_DECODE_OVER);
+
+  if (tDecodeSMqOffsetObj(buf, pOffset) == NULL) {
+    goto OFFSET_DECODE_OVER;
+  }
+
+  terrno = TSDB_CODE_SUCCESS;
+
+OFFSET_DECODE_OVER:
+  tfree(buf);
+  if (terrno != TSDB_CODE_SUCCESS) {
+    mError("offset:%s, failed to decode from raw:%p since %s", pOffset->key, pRaw, terrstr());
+    tfree(pRow);
+    return NULL;
+  }
+
+  mTrace("offset:%s, decode from raw:%p, row:%p", pOffset->key, pRaw, pOffset);
+  return pRow;
+}
+
+int32_t mndCreateOffset(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs) {
+  int32_t code = 0;
+  int32_t sz = taosArrayGetSize(vgs);
+  for (int32_t i = 0; i < sz; i++) {
+    SMqConsumerEp *pConsumerEp = taosArrayGet(vgs, i);
+    SMqOffsetObj   offsetObj;
+    if (mndMakePartitionKey(offsetObj.key, cgroup, topicName, pConsumerEp->vgId) < 0) {
+      return -1;
+    }
+    offsetObj.offset = -1;
+    SSdbRaw *pOffsetRaw = mndOffsetActionEncode(&offsetObj);
+    if (pOffsetRaw == NULL) {
+      return -1;
+    }
+    sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY);
+    if (mndTransAppendRedolog(pTrans, pOffsetRaw) < 0) {
+      return -1;
+    }
+  }
+  return 0;
+}
+
+static int32_t mndProcessCommitOffsetReq(SMnodeMsg *pMsg) {
+  char key[TSDB_PARTITION_KEY_LEN];
+
+  SMnode              *pMnode = pMsg->pMnode;
+  char                *msgStr = pMsg->rpcMsg.pCont;
+  SMqCMCommitOffsetReq commitOffsetReq;
+  SCoder               decoder;
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, msgStr, pMsg->rpcMsg.contLen, TD_DECODER);
+
+  tDecodeSMqCMCommitOffsetReq(&decoder, &commitOffsetReq);
+
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_COMMIT_OFFSET, &pMsg->rpcMsg);
+
+  for (int32_t i = 0; i < commitOffsetReq.num; i++) {
+    SMqOffset *pOffset = &commitOffsetReq.offsets[i];
+    if (mndMakePartitionKey(key, pOffset->cgroup, pOffset->topicName, pOffset->vgId) < 0) {
+      return -1;
+    }
+    SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, key);
+    ASSERT(pOffsetObj);
+    pOffsetObj->offset = pOffset->offset;
+    SSdbRaw *pOffsetRaw = mndOffsetActionEncode(pOffsetObj);
+    sdbSetRawStatus(pOffsetRaw, SDB_STATUS_READY);
+    mndTransAppendRedolog(pTrans, pOffsetRaw);
+    mndReleaseOffset(pMnode, pOffsetObj);
+  }
+
+  if (mndTransPrepare(pMnode, pTrans) != 0) {
+    mError("mq-commit-offset-trans:%d, failed to prepare since %s", pTrans->id, terrstr());
+    mndTransDrop(pTrans);
+    return -1;
+  }
+
+  mndTransDrop(pTrans);
+  return TSDB_CODE_MND_ACTION_IN_PROGRESS;
+}
+
+static int32_t mndOffsetActionInsert(SSdb *pSdb, SMqOffsetObj *pOffset) {
+  mTrace("offset:%s, perform insert action", pOffset->key);
+  return 0;
+}
+
+static int32_t mndOffsetActionDelete(SSdb *pSdb, SMqOffsetObj *pOffset) {
+  mTrace("offset:%s, perform delete action", pOffset->key);
+  return 0;
+}
+
+static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOldOffset, SMqOffsetObj *pNewOffset) {
+  mTrace("offset:%s, perform update action", pOldOffset->key);
+  return 0;
+}
+
+SMqOffsetObj *mndAcquireOffset(SMnode *pMnode, const char *key) {
+  SSdb         *pSdb = pMnode->pSdb;
+  SMqOffsetObj *pOffset = sdbAcquire(pSdb, SDB_OFFSET, key);
+  if (pOffset == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
+    terrno = TSDB_CODE_MND_OFFSET_NOT_EXIST;
+  }
+  return pOffset;
+}
+
+void mndReleaseOffset(SMnode *pMnode, SMqOffsetObj *pOffset) {
+  SSdb *pSdb = pMnode->pSdb;
+  sdbRelease(pSdb, pOffset);
+}
+
+static void mndCancelGetNextOffset(SMnode *pMnode, void *pIter) {
+  SSdb *pSdb = pMnode->pSdb;
+  sdbCancelFetch(pSdb, pIter);
+}

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -19,6 +19,7 @@
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndMnode.h"
+#include "mndOffset.h"
 #include "mndShow.h"
 #include "mndStb.h"
 #include "mndTopic.h"
@@ -80,13 +81,13 @@ int32_t mndInitSubscribe(SMnode *pMnode) {
   return sdbSetTable(pMnode->pSdb, table);
 }
 
-static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *consumerGroup) {
+static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *cgroup) {
   SMqSubscribeObj *pSub = tNewSubscribeObj();
   if (pSub == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
-  char *key = mndMakeSubscribeKey(consumerGroup, pTopic->name);
+  char *key = mndMakeSubscribeKey(cgroup, pTopic->name);
   if (key == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     tDeleteSMqSubscribeObj(pSub);
@@ -289,9 +290,15 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) {
           strcpy(topicEp.topic, topicName);
           topicEp.vgs = taosArrayInit(vgsz, sizeof(SMqSubVgEp));
           for (int32_t k = 0; k < vgsz; k++) {
+            char           offsetKey[TSDB_PARTITION_KEY_LEN];
             SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, k);
-
-            SMqSubVgEp vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId};
+            SMqSubVgEp     vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId, .offset = -1};
+            mndMakePartitionKey(offsetKey, pConsumer->cgroup, topicName, pConsumerEp->vgId);
+            SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, offsetKey);
+            if (pOffsetObj != NULL) {
+              vgEp.offset = pOffsetObj->offset;
+              mndReleaseOffset(pMnode, pOffsetObj);
+            }
             taosArrayPush(topicEp.vgs, &vgEp);
           }
           taosArrayPush(rsp.topics, &topicEp);
@@ -870,7 +877,7 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) {
 
 SUB_ENCODE_OVER:
   tfree(buf);
-  if (terrno != 0) {
+  if (terrno != TSDB_CODE_SUCCESS) {
     mError("subscribe:%s, failed to encode to raw:%p since %s", pSub->key, pRaw, terrstr());
     sdbFreeRaw(pRaw);
     return NULL;
@@ -1085,6 +1092,8 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) {
         mDebug("create new subscription by consumer %ld, group: %s, topic %s", consumerId, cgroup, newTopicName);
         pSub = mndCreateSubscription(pMnode, pTopic, cgroup);
         createSub = true;
+
+        mndCreateOffset(pTrans, cgroup, newTopicName, pSub->unassignedVg);
       }
 
       SMqSubConsumer mqSubConsumer;

source/dnode/mnode/impl/src/mnode.c

@@ -23,6 +23,7 @@
 #include "mndDnode.h"
 #include "mndFunc.h"
 #include "mndMnode.h"
+#include "mndOffset.h"
 #include "mndProfile.h"
 #include "mndQnode.h"
 #include "mndShow.h"
@@ -217,6 +218,7 @@ static int32_t mndInitSteps(SMnode *pMnode) {
   if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-consumer", mndInitConsumer, mndCleanupConsumer) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-subscribe", mndInitSubscribe, mndCleanupSubscribe) != 0) return -1;
+  if (mndAllocStep(pMnode, "mnode-offset", mndInitOffset, mndCleanupOffset) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1;

source/dnode/mnode/impl/test/qnode/qnode.cpp

@@ -190,6 +190,9 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) {
     tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq);
 
     server2.Stop();
+    taosMsleep(1000);
+    // test.ClientRestart();
+
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL);
@@ -226,6 +229,7 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) {
   {
     // server start, wait until the rollback finished
     server2.DoStart();
+    test.ClientRestart();
     taosMsleep(1000);
 
     int32_t retry = 0;
@@ -248,7 +252,6 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) {
     ASSERT_NE(retry, retryMax);
   }
 }
-
 TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) {
   {
     // send message first, then dnode2 crash, result is returned, and rollback is started

source/dnode/mnode/impl/test/trans/trans.cpp

@@ -46,8 +46,10 @@ class MndTestTrans : public ::testing::Test {
     free(buffer);
     taosFsyncFile(fd);
     taosCloseFile(fd);
+    taosMsleep(1000);
 
     test.ServerStart();
+    test.ClientRestart();
   }
 
   static Testbase   test;
@@ -284,6 +286,7 @@ TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) {
   }
 }
 
+
 // create db
 // partial create stb
 // drop db failed

source/dnode/mnode/impl/test/user/user.cpp

@@ -617,6 +617,7 @@ TEST_F(MndTestUser, 06_Create_Drop_Alter_User) {
   // restart
   test.Restart();
 
+  taosMsleep(1000);
   test.SendShowMetaReq(TSDB_MGMT_TABLE_USER, "");
   CHECK_META("show users", 4);
 

source/dnode/vnode/src/tq/tq.c

@@ -207,9 +207,17 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
 int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
   SMqConsumeReq* pReq = pMsg->pCont;
   int64_t        consumerId = pReq->consumerId;
-  int64_t        fetchOffset = pReq->offset;
+  int64_t        fetchOffset;
   /*int64_t        blockingTime = pReq->blockingTime;*/
 
+  if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
+    fetchOffset = 0;
+  } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) {
+    fetchOffset = walGetLastVer(pTq->pWal);
+  } else {
+    fetchOffset = pReq->currentOffset + 1;
+  }
+
   SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 0, .pBlockData = NULL};
 
   STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId);
@@ -226,31 +234,9 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
   ASSERT(strcmp(pTopic->topicName, pReq->topic) == 0);
   ASSERT(pConsumer->consumerId == consumerId);
 
-  if (pReq->reqType == TMQ_REQ_TYPE_COMMIT_ONLY) {
-    pTopic->committedOffset = pReq->offset;
-    /*printf("offset %ld committed\n", pTopic->committedOffset);*/
-    pMsg->pCont = NULL;
-    pMsg->contLen = 0;
-    pMsg->code = 0;
-    rpcSendResponse(pMsg);
-    return 0;
-  }
-
-  if (pReq->reqType == TMQ_REQ_TYPE_CONSUME_AND_COMMIT) {
-    if (pTopic->committedOffset < pReq->offset - 1) {
-      pTopic->committedOffset = pReq->offset - 1;
-      /*printf("offset %ld committed\n", pTopic->committedOffset);*/
-    }
-  }
-
-  rsp.committedOffset = pTopic->committedOffset;
-  rsp.reqOffset = pReq->offset;
+  rsp.reqOffset = pReq->currentOffset;
   rsp.skipLogNum = 0;
 
-  if (fetchOffset <= pTopic->committedOffset) {
-    fetchOffset = pTopic->committedOffset + 1;
-  }
-
   SWalHead* pHead;
   while (1) {
     int8_t pos = fetchOffset % TQ_BUFFER_SIZE;

source/dnode/vnode/src/vnd/vnodeQuery.c

@@ -126,7 +126,7 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) {
     goto _exit;
   }
 
-  metaRsp.dbId = htobe64(pVnode->config.dbId);
+  metaRsp.dbId = pVnode->config.dbId;
   memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName));
   strcpy(metaRsp.tbName, infoReq.tbName);
   if (pTbCfg->type == META_CHILD_TABLE) {

source/dnode/vnode/src/vnd/vnodeWrite.c

@@ -22,14 +22,15 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
   for (int i = 0; i < taosArrayGetSize(pMsgs); i++) {
     pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i);
 
-    // ser request version
+    // set request version
     void   *pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
     int64_t ver = pVnode->state.processed++;
     taosEncodeFixedI64(&pBuf, ver);
 
     if (walWrite(pVnode->pWal, ver, pMsg->msgType, pMsg->pCont, pMsg->contLen) < 0) {
-      /*ASSERT(false);*/
       // TODO: handle error
+      /*ASSERT(false);*/
+      vError("vnode:%d  write wal error since %s", pVnode->vgId, terrstr());
     }
   }
 

source/libs/CMakeLists.txt

@@ -15,3 +15,4 @@ add_subdirectory(qworker)
 add_subdirectory(tfs)
 add_subdirectory(nodes)
 add_subdirectory(config)
+add_subdirectory(scalar)

source/libs/catalog/src/catalog.c

@@ -546,7 +546,7 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
 
   if (tbMeta->tableType != TSDB_CHILD_TABLE) {
     ctgReleaseDBCache(pCtg, dbCache);
-    ctgDebug("Got tbl from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname);
+    ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, pTableName->tname);
     return TSDB_CODE_SUCCESS;
   }
   
@@ -1556,6 +1556,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
 
   STableMetaOutput *output = NULL;
 
+  while (true) {
     CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, isSTable, &output));
 
     if (CTG_IS_META_TABLE(output->metaType)) {
@@ -1584,11 +1585,14 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
     CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist));
     if (0 == exist) {
       ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname);
-    CTG_ERR_JRET(TSDB_CODE_VND_TB_NOT_EXIST);
+      continue;
     }
 
     memcpy(*pTableMeta, &output->ctbMeta, sizeof(output->ctbMeta));
 
+    break;
+  }
+
 _return:
 
   tfree(output);

source/libs/executor/CMakeLists.txt

@@ -13,7 +13,7 @@ add_library(executor STATIC ${EXECUTOR_SRC})
 #        INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/executor"
 #        )
 target_link_libraries(executor
-            PRIVATE os util common function parser planner qcom vnode
+            PRIVATE os util common function parser planner qcom vnode scalar nodes
         )
 
 target_include_directories(

source/libs/executor/inc/executorimpl.h

@@ -29,9 +29,9 @@ extern "C" {
 #include "executil.h"
 #include "executor.h"
 #include "planner.h"
+#include "scalar.h"
 #include "taosdef.h"
 #include "tarray.h"
-#include "tfilter.h"
 #include "thash.h"
 #include "tlockfree.h"
 #include "tpagedbuf.h"

source/libs/executor/src/executor.c

@@ -35,7 +35,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, char* id)
   } else {
     SStreamBlockScanInfo* pInfo = pOperator->info;
     if (tqReadHandleSetMsg(pInfo->readerHandle, input, 0) < 0) {
-      qError("submit msg error while set stream msg, %s" PRIx64, id);
+      qError("submit msg messed up when initing stream block, %s" PRIx64, id);
       return TSDB_CODE_QRY_APP_ERROR;
     }
     return TSDB_CODE_SUCCESS;

source/libs/executor/src/executorimpl.c

@@ -2148,7 +2148,7 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT
   // NOTE: pTableCheckInfo need to update the query time range and the lastKey info
   pRuntimeEnv->pTableRetrieveTsMap = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
 
-  pRuntimeEnv->scalarSup = createScalarFuncSupport(pQueryAttr->numOfOutput);
+  //pRuntimeEnv->scalarSup = createScalarFuncSupport(pQueryAttr->numOfOutput);
 
   if (pRuntimeEnv->scalarSup == NULL || pRuntimeEnv->pResultRowHashTable == NULL || pRuntimeEnv->keyBuf == NULL ||
       pRuntimeEnv->prevRow == NULL  || pRuntimeEnv->tagVal == NULL) {
@@ -2174,7 +2174,7 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT
   return TSDB_CODE_SUCCESS;
 
 _clean:
-  destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pRuntimeEnv->pQueryAttr->numOfOutput);
+  //destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pRuntimeEnv->pQueryAttr->numOfOutput);
   tfree(pRuntimeEnv->pResultRowHashTable);
   tfree(pRuntimeEnv->keyBuf);
   tfree(pRuntimeEnv->prevRow);
@@ -2212,7 +2212,7 @@ static void teardownQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv) {
 
   //qDebug("QInfo:0x%"PRIx64" teardown runtime env", pQInfo->qId);
 
-  destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput);
+  //destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput);
 //  destroyUdfInfo(pRuntimeEnv->pUdfInfo);
   destroyResultBuf(pRuntimeEnv->pResultBuf);
   doFreeQueryHandle(pRuntimeEnv);

source/libs/executor/test/CMakeLists.txt

@@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 ADD_EXECUTABLE(executorTest ${SOURCE_LIST})
 TARGET_LINK_LIBRARIES(
         executorTest
-        PUBLIC os util common transport gtest taos qcom executor function planner
+        PRIVATE os util common transport gtest taos qcom executor function planner scalar nodes
 )
 
 TARGET_INCLUDE_DIRECTORIES(

source/libs/executor/test/executorTests.cpp

@@ -32,6 +32,8 @@
 #include "trpc.h"
 #include "stub.h"
 #include "executor.h"
+#include "tmsg.h"
+#include "tname.h"
 
 namespace {
 

source/libs/function/inc/builtins.h

@@ -46,6 +46,7 @@ typedef struct SBuiltinFuncDefinition {
   FExecGetEnv getEnvFunc;
   FExecInit initFunc;
   FExecProcess processFunc;
+  FScalarExecProcess sprocessFunc;
   FExecFinalize finalizeFunc;
 } SBuiltinFuncDefinition;
 

source/libs/function/inc/tunaryoperator.h

@@ -20,10 +20,10 @@
 extern "C" {
 #endif
 
-#include "tscalarfunction.h"
+//#include "tscalarfunction.h"
 
-typedef void (*_unary_scalar_fn_t)(SScalarFuncParam *pLeft, SScalarFuncParam* pOutput);
-_unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t binOperator);
+//typedef void (*_unary_scalar_fn_t)(SScalarParam *pLeft, SScalarParam* pOutput);
+//_unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t binOperator);
 
 #ifdef __cplusplus
 }

source/libs/function/src/builtins.c

@@ -36,7 +36,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .checkFunc = stubCheckAndGetResultType,
     .getEnvFunc = NULL,
     .initFunc = NULL,
-    .processFunc = NULL,
+    .sprocessFunc = NULL,
     .finalizeFunc = NULL
   }
 };

source/libs/function/src/functionMgt.c

@@ -71,6 +71,14 @@ int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet) {
+  if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
+    return TSDB_CODE_FAILED;
+  }
+  pFpSet->process = funcMgtBuiltins[funcId].sprocessFunc;
+  return TSDB_CODE_SUCCESS;
+}
+
 bool fmIsAggFunc(int32_t funcId) {
   if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
     return false;

source/libs/function/src/taggfunction.c

@@ -13,7 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tscalarfunction.h"
 #include "os.h"
 #include "taosdef.h"
 #include "tmsg.h"
@@ -3221,6 +3220,7 @@ static void diff_function(SqlFunctionCtx *pCtx) {
   }
 }
 
+#if 0
 char *getArithColumnData(void *param, const char* name, int32_t colId) {
   SScalarFunctionSupport *pSupport = (SScalarFunctionSupport *)param;
   
@@ -3235,15 +3235,16 @@ char *getArithColumnData(void *param, const char* name, int32_t colId) {
   assert(index >= 0);
   return pSupport->data[index] + pSupport->offset * pSupport->colList[index].bytes;
 }
+#endif
 
 static void arithmetic_function(SqlFunctionCtx *pCtx) {
   GET_RES_INFO(pCtx)->numOfRes += pCtx->size;
-  SScalarFunctionSupport *pSup = (SScalarFunctionSupport *)pCtx->param[1].pz;
+  //SScalarFunctionSupport *pSup = (SScalarFunctionSupport *)pCtx->param[1].pz;
 
-  SScalarFuncParam output = {0};
+  SScalarParam output = {0};
   output.data = pCtx->pOutput;
 
-  evaluateExprNodeTree(pSup->pExprInfo->pExpr, pCtx->size, &output, pSup, getArithColumnData);
+  //evaluateExprNodeTree(pSup->pExprInfo->pExpr, pCtx->size, &output, pSup, getArithColumnData);
 }
 
 #define LIST_MINMAX_N(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType, numOfNotNullElem) \

source/libs/function/src/texpr.c

@@ -25,6 +25,7 @@
 #include "thash.h"
 #include "texpr.h"
 #include "tvariant.h"
+#include "tdef.h"
 
 //static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) {
 //  if (pLeft->nodeType == TEXPR_COL_NODE) {
@@ -94,7 +95,7 @@ bool exprTreeApplyFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp
 
   //non-leaf nodes, recursively traverse the expression tree in the post-root order
   if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE && pRight->nodeType == TEXPR_BINARYEXPR_NODE) {
-    if (pExpr->_node.optr == TSDB_RELATION_OR) {  // or
+    if (pExpr->_node.optr == LOGIC_COND_TYPE_OR) {  // or
       if (exprTreeApplyFilter(pLeft, pItem, param)) {
         return true;
       }
@@ -255,7 +256,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
 
   if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN) == 0) {
     right->nodeType = TEXPR_VALUE_NODE;
-    expr->_node.optr = TSDB_RELATION_LIKE;
+    expr->_node.optr = OP_TYPE_LIKE;
     SVariant* pVal = exception_calloc(1, sizeof(SVariant));
     right->pVal = pVal;
     size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_LIKE_LEN) + 1;
@@ -266,7 +267,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
 
   } else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_MATCH, QUERY_COND_REL_PREFIX_MATCH_LEN) == 0) {
     right->nodeType = TEXPR_VALUE_NODE;
-    expr->_node.optr = TSDB_RELATION_MATCH;
+    expr->_node.optr = OP_TYPE_MATCH;
     SVariant* pVal = exception_calloc(1, sizeof(SVariant));
     right->pVal = pVal;
     size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_MATCH_LEN) + 1;
@@ -276,7 +277,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
     pVal->nLen = (int32_t)len;
   } else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_NMATCH, QUERY_COND_REL_PREFIX_NMATCH_LEN) == 0) {
     right->nodeType = TEXPR_VALUE_NODE;
-    expr->_node.optr = TSDB_RELATION_NMATCH;
+    expr->_node.optr = OP_TYPE_NMATCH;
     SVariant* pVal = exception_calloc(1, sizeof(SVariant));
     right->pVal = pVal;
     size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_NMATCH_LEN) + 1;
@@ -286,7 +287,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
     pVal->nLen = (int32_t)len;
   } else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN) == 0) {
     right->nodeType = TEXPR_VALUE_NODE;
-    expr->_node.optr = TSDB_RELATION_IN;
+    expr->_node.optr = OP_TYPE_IN;
     SVariant* pVal = exception_calloc(1, sizeof(SVariant));
     right->pVal = pVal;
     pVal->nType = TSDB_DATA_TYPE_POINTER_ARRAY;

source/libs/function/src/tfunction.c

@@ -3,7 +3,6 @@
 #include "function.h"
 #include "thash.h"
 #include "taggfunction.h"
-#include "tscalarfunction.h"
 
 static SHashObj* functionHashTable = NULL;
 static SHashObj* udfHashTable = NULL;
@@ -18,12 +17,14 @@ static void doInitFunctionHashTable() {
     taosHashPut(functionHashTable, aggFunc[i].name, len, (void*)&ptr, POINTER_BYTES);
   }
 
+/*
   numOfEntries = tListLen(scalarFunc);
   for(int32_t i = 0; i < numOfEntries; ++i) {
     int32_t len = (int32_t) strlen(scalarFunc[i].name);
     SScalarFunctionInfo* ptr = &scalarFunc[i];
     taosHashPut(functionHashTable, scalarFunc[i].name, len, (void*)&ptr, POINTER_BYTES);
   }
+*/
 
   udfHashTable = taosHashInit(numOfEntries, MurmurHash3_32, true, true);
 }

source/libs/function/src/tunaryoperator.c

@@ -4,10 +4,10 @@
 
 
 // TODO dynamic define these functions
-_unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t operator) {
-  assert(0);
-}
+//_unary_scalar_fn_t getUnaryScalarOperatorFn(int32_t operator) {
+//  assert(0);
+//}
 
-bool isStringOperatorFn(int32_t op) {
-  return op == FUNCTION_LENGTH;
-}
+//bool isStringOperatorFn(int32_t op) {
+//  return op == FUNCTION_LENGTH;
+//}

source/libs/index/src/index.c

@@ -66,7 +66,9 @@ static void indexMergeSameKey(SArray* result, TFileValue* tv);
 int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
   pthread_once(&isInit, indexInit);
   SIndex* sIdx = calloc(1, sizeof(SIndex));
-  if (sIdx == NULL) { return -1; }
+  if (sIdx == NULL) {
+    return -1;
+  }
 
 #ifdef USE_LUCENE
   index_t* index = index_open(path);
@@ -76,7 +78,9 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
 #ifdef USE_INVERTED_INDEX
   // sIdx->cache = (void*)indexCacheCreate(sIdx);
   sIdx->tindex = indexTFileCreate(path);
-  if (sIdx->tindex == NULL) { goto END; }
+  if (sIdx->tindex == NULL) {
+    goto END;
+  }
 
   sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
   sIdx->cVersion = 1;
@@ -87,7 +91,9 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
 #endif
 
 END:
-  if (sIdx != NULL) { indexClose(sIdx); }
+  if (sIdx != NULL) {
+    indexClose(sIdx);
+  }
 
   *index = NULL;
   return -1;
@@ -103,7 +109,9 @@ void indexClose(SIndex* sIdx) {
   void* iter = taosHashIterate(sIdx->colObj, NULL);
   while (iter) {
     IndexCache** pCache = iter;
-    if (*pCache) { indexCacheUnRef(*pCache); }
+    if (*pCache) {
+      indexCacheUnRef(*pCache);
+    }
     iter = taosHashIterate(sIdx->colObj, iter);
   }
   taosHashCleanup(sIdx->colObj);
@@ -161,7 +169,9 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
     IndexCache** cache = taosHashGet(index->colObj, buf, sz);
     assert(*cache != NULL);
     int ret = indexCachePut(*cache, p, uid);
-    if (ret != 0) { return ret; }
+    if (ret != 0) {
+      return ret;
+    }
   }
 
 #endif
@@ -191,7 +201,9 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result
   int  tsz = 0;
   index_multi_search(index->index, (const char**)fields, (const char**)keys, types, nQuery, opera, &tResult, &tsz);
 
-  for (int i = 0; i < tsz; i++) { taosArrayPush(result, &tResult[i]); }
+  for (int i = 0; i < tsz; i++) {
+    taosArrayPush(result, &tResult[i]);
+  }
 
   for (int i = 0; i < nQuery; i++) {
     free(fields[i]);
@@ -248,7 +260,9 @@ void indexOptsDestroy(SIndexOpts* opts) {
  */
 SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) {
   SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery));
-  if (p == NULL) { return NULL; }
+  if (p == NULL) {
+    return NULL;
+  }
   p->opera = opera;
   p->query = taosArrayInit(4, sizeof(SIndexTermQuery));
   return p;
@@ -270,7 +284,9 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde
 SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName,
                             int32_t nColName, const char* colVal, int32_t nColVal) {
   SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm)));
-  if (t == NULL) { return NULL; }
+  if (t == NULL) {
+    return NULL;
+  }
 
   t->suid = suid;
   t->operType = oper;
@@ -343,7 +359,9 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result
   return 0;
 }
 static void indexInterResultsDestroy(SArray* results) {
-  if (results == NULL) { return; }
+  if (results == NULL) {
+    return;
+  }
 
   size_t sz = taosArrayGetSize(results);
   for (size_t i = 0; i < sz; i++) {
@@ -419,18 +437,24 @@ static void indexDestroyTempResult(SArray* result) {
   taosArrayDestroy(result);
 }
 int indexFlushCacheToTFile(SIndex* sIdx, void* cache) {
-  if (sIdx == NULL) { return -1; }
+  if (sIdx == NULL) {
+    return -1;
+  }
   indexInfo("suid %" PRIu64 " merge cache into tindex", sIdx->suid);
 
   int64_t st = taosGetTimestampUs();
 
   IndexCache*  pCache = (IndexCache*)cache;
   TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->suid, pCache->colName);
-  if (pReader == NULL) { indexWarn("empty tfile reader found"); }
+  if (pReader == NULL) {
+    indexWarn("empty tfile reader found");
+  }
   // handle flush
   Iterate* cacheIter = indexCacheIteratorCreate(pCache);
   Iterate* tfileIter = tfileIteratorCreate(pReader);
-  if (tfileIter == NULL) { indexWarn("empty tfile reader iterator"); }
+  if (tfileIter == NULL) {
+    indexWarn("empty tfile reader iterator");
+  }
 
   SArray* result = taosArrayInit(1024, sizeof(void*));
 
@@ -484,7 +508,9 @@ void iterateValueDestroy(IterateValue* value, bool destroy) {
     taosArrayDestroy(value->val);
     value->val = NULL;
   } else {
-    if (value->val != NULL) { taosArrayClear(value->val); }
+    if (value->val != NULL) {
+      taosArrayClear(value->val);
+    }
   }
   free(value->colVal);
   value->colVal = NULL;
@@ -507,7 +533,9 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) {
   tfileWriterClose(tw);
 
   TFileReader* reader = tfileReaderOpen(sIdx->path, cache->suid, version, cache->colName);
-  if (reader == NULL) { return -1; }
+  if (reader == NULL) {
+    return -1;
+  }
 
   TFileHeader* header = &reader->header;
   ICacheKey    key = {.suid = cache->suid, .colName = header->colName, .nColName = strlen(header->colName)};

source/libs/index/src/index_cache.c

@@ -119,13 +119,17 @@ void indexCacheDestroySkiplist(SSkipList* slt) {
   tSkipListDestroy(slt);
 }
 void indexCacheDestroyImm(IndexCache* cache) {
-  if (cache == NULL) { return; }
+  if (cache == NULL) {
+    return;
+  }
 
   MemTable* tbl = NULL;
   pthread_mutex_lock(&cache->mtx);
+
   tbl = cache->imm;
   cache->imm = NULL;  // or throw int bg thread
   pthread_cond_broadcast(&cache->finished);
+
   pthread_mutex_unlock(&cache->mtx);
 
   indexMemUnRef(tbl);
@@ -133,7 +137,9 @@ void indexCacheDestroyImm(IndexCache* cache) {
 }
 void indexCacheDestroy(void* cache) {
   IndexCache* pCache = cache;
-  if (pCache == NULL) { return; }
+  if (pCache == NULL) {
+    return;
+  }
   indexMemUnRef(pCache->mem);
   indexMemUnRef(pCache->imm);
   free(pCache->colName);
@@ -146,7 +152,9 @@ void indexCacheDestroy(void* cache) {
 
 Iterate* indexCacheIteratorCreate(IndexCache* cache) {
   Iterate* iiter = calloc(1, sizeof(Iterate));
-  if (iiter == NULL) { return NULL; }
+  if (iiter == NULL) {
+    return NULL;
+  }
 
   pthread_mutex_lock(&cache->mtx);
 
@@ -164,7 +172,9 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) {
   return iiter;
 }
 void indexCacheIteratorDestroy(Iterate* iter) {
-  if (iter == NULL) { return; }
+  if (iter == NULL) {
+    return;
+  }
   tSkipListDestroyIter(iter->iter);
   iterateValueDestroy(&iter->val, true);
   free(iter);
@@ -186,9 +196,6 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) {
     } else if (cache->imm != NULL) {
       // TODO: wake up by condition variable
       pthread_cond_wait(&cache->finished, &cache->mtx);
-      // pthread_mutex_unlock(&cache->mtx);
-      // taosMsleep(50);
-      // pthread_mutex_lock(&cache->mtx);
     } else {
       indexCacheRef(cache);
       cache->imm = cache->mem;
@@ -202,13 +209,17 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) {
 }
 
 int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) {
-  if (cache == NULL) { return -1; }
+  if (cache == NULL) {
+    return -1;
+  }
 
   IndexCache* pCache = cache;
   indexCacheRef(pCache);
   // encode data
   CacheTerm* ct = calloc(1, sizeof(CacheTerm));
-  if (cache == NULL) { return -1; }
+  if (cache == NULL) {
+    return -1;
+  }
   // set up key
   ct->colType = term->colType;
   ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1));
@@ -240,7 +251,9 @@ int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t u
 }
 
 static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SArray* result, STermValueType* s) {
-  if (mem == NULL) { return 0; }
+  if (mem == NULL) {
+    return 0;
+  }
   char* key = indexCacheTermGet(ct);
 
   SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
@@ -266,7 +279,9 @@ static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SA
   return 0;
 }
 int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermValueType* s) {
-  if (cache == NULL) { return 0; }
+  if (cache == NULL) {
+    return 0;
+  }
   IndexCache* pCache = cache;
 
   MemTable *mem = NULL, *imm = NULL;
@@ -294,23 +309,33 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermV
 }
 
 void indexCacheRef(IndexCache* cache) {
-  if (cache == NULL) { return; }
+  if (cache == NULL) {
+    return;
+  }
   int ref = T_REF_INC(cache);
   UNUSED(ref);
 }
 void indexCacheUnRef(IndexCache* cache) {
-  if (cache == NULL) { return; }
+  if (cache == NULL) {
+    return;
+  }
   int ref = T_REF_DEC(cache);
-  if (ref == 0) { indexCacheDestroy(cache); }
+  if (ref == 0) {
+    indexCacheDestroy(cache);
+  }
 }
 
 void indexMemRef(MemTable* tbl) {
-  if (tbl == NULL) { return; }
+  if (tbl == NULL) {
+    return;
+  }
   int ref = T_REF_INC(tbl);
   UNUSED(ref);
 }
 void indexMemUnRef(MemTable* tbl) {
-  if (tbl == NULL) { return; }
+  if (tbl == NULL) {
+    return;
+  }
   int ref = T_REF_DEC(tbl);
   if (ref == 0) {
     SSkipList* slt = tbl->mem;
@@ -320,7 +345,9 @@ void indexMemUnRef(MemTable* tbl) {
 }
 
 static void indexCacheTermDestroy(CacheTerm* ct) {
-  if (ct == NULL) { return; }
+  if (ct == NULL) {
+    return;
+  }
   free(ct->colVal);
   free(ct);
 }
@@ -333,7 +360,9 @@ static int32_t indexCacheTermCompare(const void* l, const void* r) {
   CacheTerm* rt = (CacheTerm*)r;
   // compare colVal
   int32_t cmp = strcmp(lt->colVal, rt->colVal);
-  if (cmp == 0) { return rt->version - lt->version; }
+  if (cmp == 0) {
+    return rt->version - lt->version;
+  }
   return cmp;
 }
 
@@ -354,7 +383,9 @@ static void doMergeWork(SSchedMsg* msg) {
 }
 static bool indexCacheIteratorNext(Iterate* itera) {
   SSkipListIterator* iter = itera->iter;
-  if (iter == NULL) { return false; }
+  if (iter == NULL) {
+    return false;
+  }
   IterateValue* iv = &itera->val;
   iterateValueDestroy(iv, false);
 

source/libs/index/src/index_fst.c

@@ -31,20 +31,24 @@ static uint8_t fstPackDetla(FstCountingWriter* wrt, CompiledAddr nodeAddr, Compi
 
 FstUnFinishedNodes* fstUnFinishedNodesCreate() {
   FstUnFinishedNodes* nodes = malloc(sizeof(FstUnFinishedNodes));
-  if (nodes == NULL) { return NULL; }
+  if (nodes == NULL) {
+    return NULL;
+  }
 
   nodes->stack = (SArray*)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished));
   fstUnFinishedNodesPushEmpty(nodes, false);
   return nodes;
 }
-void unFinishedNodeDestroyElem(void* elem) {
+static void unFinishedNodeDestroyElem(void* elem) {
   FstBuilderNodeUnfinished* b = (FstBuilderNodeUnfinished*)elem;
   fstBuilderNodeDestroy(b->node);
   free(b->last);
   b->last = NULL;
 }
 void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) {
-  if (nodes == NULL) { return; }
+  if (nodes == NULL) {
+    return;
+  }
 
   taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem);
   free(nodes);
@@ -92,7 +96,9 @@ void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes* nodes, CompiledAddr add
 }
 void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output out) {
   FstSlice* s = &bs;
-  if (fstSliceIsEmpty(s)) { return; }
+  if (fstSliceIsEmpty(s)) {
+    return;
+  }
   size_t                    sz = taosArrayGetSize(nodes->stack) - 1;
   FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz);
   assert(un->last == NULL);
@@ -172,7 +178,9 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes* node,
 
 FstState fstStateCreateFrom(FstSlice* slice, CompiledAddr addr) {
   FstState fs = {.state = EmptyFinal, .val = 0};
-  if (addr == EMPTY_ADDRESS) { return fs; }
+  if (addr == EMPTY_ADDRESS) {
+    return fs;
+  }
 
   uint8_t* data = fstSliceData(slice, NULL);
   uint8_t  v = data[addr];
@@ -229,7 +237,9 @@ void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTran
   fstStateSetCommInput(&st, trn->inp);
   bool    null = false;
   uint8_t inp = fstStateCommInput(&st, &null);
-  if (null == true) { fstCountingWriterWrite(w, (char*)&trn->inp, sizeof(trn->inp)); }
+  if (null == true) {
+    fstCountingWriterWrite(w, (char*)&trn->inp, sizeof(trn->inp));
+  }
   fstCountingWriterWrite(w, (char*)(&(st.val)), sizeof(st.val));
   return;
 }
@@ -263,7 +273,9 @@ void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuil
   fstStateSetStateNtrans(&st, (uint8_t)sz);
 
   if (anyOuts) {
-    if (FST_BUILDER_NODE_IS_FINAL(node)) { fstCountingWriterPackUintIn(w, node->finalOutput, oSize); }
+    if (FST_BUILDER_NODE_IS_FINAL(node)) {
+      fstCountingWriterPackUintIn(w, node->finalOutput, oSize);
+    }
     for (int32_t i = sz - 1; i >= 0; i--) {
       FstTransition* t = taosArrayGet(node->trans, i);
       fstCountingWriterPackUintIn(w, t->out, oSize);
@@ -428,7 +440,9 @@ Output fstStateOutput(FstState* s, FstNode* node) {
   assert(s->state == OneTrans);
 
   uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes);
-  if (oSizes == 0) { return 0; }
+  if (oSizes == 0) {
+    return 0;
+  }
   FstSlice* slice = &node->data;
   uint8_t   tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes);
 
@@ -440,7 +454,9 @@ Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) {
   assert(s->state == AnyTrans);
 
   uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes);
-  if (oSizes == 0) { return 0; }
+  if (oSizes == 0) {
+    return 0;
+  }
   FstSlice* slice = &node->data;
   uint8_t*  data = fstSliceData(slice, NULL);
   uint64_t  at = node->start - fstStateNtransLen(s) - 1  // pack size
@@ -453,7 +469,9 @@ Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) {
 
 void fstStateSetFinalState(FstState* s, bool yes) {
   assert(s->state == AnyTrans);
-  if (yes) { s->val |= 0b01000000; }
+  if (yes) {
+    s->val |= 0b01000000;
+  }
   return;
 }
 bool fstStateIsFinalState(FstState* s) {
@@ -463,7 +481,9 @@ bool fstStateIsFinalState(FstState* s) {
 
 void fstStateSetStateNtrans(FstState* s, uint8_t n) {
   assert(s->state == AnyTrans);
-  if (n <= 0b00111111) { s->val = (s->val & 0b11000000) | n; }
+  if (n <= 0b00111111) {
+    s->val = (s->val & 0b11000000) | n;
+  }
   return;
 }
 // state_ntrans
@@ -495,7 +515,9 @@ uint64_t fstStateNtransLen(FstState* s) {
 uint64_t fstStateNtrans(FstState* s, FstSlice* slice) {
   bool    null = false;
   uint8_t n = fstStateStateNtrans(s, &null);
-  if (null != true) { return n; }
+  if (null != true) {
+    return n;
+  }
   int32_t  len;
   uint8_t* data = fstSliceData(slice, &len);
   n = data[len - 2];
@@ -505,7 +527,9 @@ uint64_t fstStateNtrans(FstState* s, FstSlice* slice) {
 }
 Output fstStateFinalOutput(FstState* s, uint64_t version, FstSlice* slice, PackSizes sizes, uint64_t nTrans) {
   uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes);
-  if (oSizes == 0 || !fstStateIsFinalState(s)) { return 0; }
+  if (oSizes == 0 || !fstStateIsFinalState(s)) {
+    return 0;
+  }
 
   uint64_t at = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1  // pack size
                 - fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes;
@@ -522,7 +546,9 @@ uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) {
     uint8_t* data = fstSliceData(slice, &dlen);
     uint64_t i = data[at + b];
     // uint64_t i = slice->data[slice->start + at + b];
-    if (i >= node->nTrans) { *null = true; }
+    if (i >= node->nTrans) {
+      *null = true;
+    }
     return i;
   } else {
     uint64_t start = node->start - fstStateNtransLen(s) - 1  // pack size
@@ -539,7 +565,9 @@ uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) {
         return node->nTrans - i - 1;  // bug
       }
     }
-    if (i == len) { *null = true; }
+    if (i == len) {
+      *null = true;
+    }
     fstSliceDestroy(&t);
   }
 }
@@ -548,7 +576,9 @@ uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) {
 
 FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* slice) {
   FstNode* n = (FstNode*)malloc(sizeof(FstNode));
-  if (n == NULL) { return NULL; }
+  if (n == NULL) {
+    return NULL;
+  }
 
   FstState st = fstStateCreateFrom(slice, addr);
 
@@ -614,7 +644,9 @@ void fstNodeDestroy(FstNode* node) {
 }
 FstTransitions* fstNodeTransitions(FstNode* node) {
   FstTransitions* t = malloc(sizeof(FstTransitions));
-  if (NULL == t) { return NULL; }
+  if (NULL == t) {
+    return NULL;
+  }
   FstRange range = {.start = 0, .end = FST_NODE_LEN(node)};
   t->range = range;
   t->node = node;
@@ -721,7 +753,9 @@ bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, Compiled
 
 FstBuilder* fstBuilderCreate(void* w, FstType ty) {
   FstBuilder* b = malloc(sizeof(FstBuilder));
-  if (NULL == b) { return b; }
+  if (NULL == b) {
+    return b;
+  }
 
   b->wrt = fstCountingWriterCreate(w);
   b->unfinished = fstUnFinishedNodesCreate();
@@ -735,15 +769,17 @@ FstBuilder* fstBuilderCreate(void* w, FstType ty) {
   taosEncodeFixedU64(&pBuf64, VERSION);
   fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64));
 
-  memset(buf64, 0, sizeof(buf64));
   pBuf64 = buf64;
+  memset(buf64, 0, sizeof(buf64));
   taosEncodeFixedU64(&pBuf64, ty);
   fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64));
 
   return b;
 }
 void fstBuilderDestroy(FstBuilder* b) {
-  if (b == NULL) { return; }
+  if (b == NULL) {
+    return;
+  }
 
   fstCountingWriterDestroy(b->wrt);
   fstUnFinishedNodesDestroy(b->unfinished);
@@ -830,6 +866,7 @@ void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate) {
   fstUnFinishedNodesTopLastFreeze(b->unfinished, addr);
   return;
 }
+
 CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn) {
   if (FST_BUILDER_NODE_IS_FINAL(bn) && FST_BUILDER_NODE_TRANS_ISEMPTY(bn) && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn)) {
     return EMPTY_ADDRESS;
@@ -844,7 +881,9 @@ CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn) {
 
   fstBuilderNodeCompileTo(bn, b->wrt, b->lastAddr, startAddr);
   b->lastAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt) - 1);
-  if (entry->state == NOTFOUND) { FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr); }
+  if (entry->state == NOTFOUND) {
+    FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr);
+  }
   fstRegistryEntryDestroy(entry);
 
   return b->lastAddr;
@@ -887,7 +926,9 @@ FstSlice fstNodeAsSlice(FstNode* node) {
 
 FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) {
   FstLastTransition* trn = malloc(sizeof(FstLastTransition));
-  if (trn == NULL) { return NULL; }
+  if (trn == NULL) {
+    return NULL;
+  }
 
   trn->inp = inp;
   trn->out = out;
@@ -897,7 +938,9 @@ FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) {
 void fstLastTransitionDestroy(FstLastTransition* trn) { free(trn); }
 void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) {
   FstLastTransition* trn = unNode->last;
-  if (trn == NULL) { return; }
+  if (trn == NULL) {
+    return;
+  }
   FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr};
   taosArrayPush(unNode->node->trans, &t);
   fstLastTransitionDestroy(trn);
@@ -906,27 +949,35 @@ void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, Comp
 }
 
 void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished* unNode, Output out) {
-  if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) { unNode->node->finalOutput += out; }
+  if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) {
+    unNode->node->finalOutput += out;
+  }
   size_t sz = taosArrayGetSize(unNode->node->trans);
   for (size_t i = 0; i < sz; i++) {
     FstTransition* trn = taosArrayGet(unNode->node->trans, i);
     trn->out += out;
   }
-  if (unNode->last) { unNode->last->out += out; }
+  if (unNode->last) {
+    unNode->last->out += out;
+  }
   return;
 }
 
 Fst* fstCreate(FstSlice* slice) {
   int32_t slen;
   char*   buf = fstSliceData(slice, &slen);
-  if (slen < 36) { return NULL; }
+  if (slen < 36) {
+    return NULL;
+  }
   uint64_t len = slen;
   uint64_t skip = 0;
 
   uint64_t version;
   taosDecodeFixedU64(buf, &version);
   skip += sizeof(version);
-  if (version == 0 || version > VERSION) { return NULL; }
+  if (version == 0 || version > VERSION) {
+    return NULL;
+  }
 
   uint64_t type;
   taosDecodeFixedU64(buf + skip, &type);
@@ -949,10 +1000,14 @@ Fst* fstCreate(FstSlice* slice) {
   taosDecodeFixedU64(buf + len, &fstLen);
   // TODO(validate root addr)
   Fst* fst = (Fst*)calloc(1, sizeof(Fst));
-  if (fst == NULL) { return NULL; }
+  if (fst == NULL) {
+    return NULL;
+  }
 
   fst->meta = (FstMeta*)malloc(sizeof(FstMeta));
-  if (NULL == fst->meta) { goto FST_CREAT_FAILED; }
+  if (NULL == fst->meta) {
+    goto FST_CREAT_FAILED;
+  }
 
   fst->meta->version = version;
   fst->meta->rootAddr = rootAddr;
@@ -983,7 +1038,7 @@ void fstDestroy(Fst* fst) {
 
 bool fstGet(Fst* fst, FstSlice* b, Output* out) {
   // dec lock range
-  pthread_mutex_lock(&fst->mtx);
+  // pthread_mutex_lock(&fst->mtx);
   FstNode* root = fstGetRoot(fst);
   Output   tOut = 0;
   int32_t  len;
@@ -996,7 +1051,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) {
     uint8_t inp = data[i];
     Output  res = 0;
     if (false == fstNodeFindInput(root, inp, &res)) {
-      pthread_mutex_unlock(&fst->mtx);
+      // pthread_mutex_unlock(&fst->mtx);
       return false;
     }
 
@@ -1007,7 +1062,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) {
     taosArrayPush(nodes, &root);
   }
   if (!FST_NODE_IS_FINAL(root)) {
-    pthread_mutex_unlock(&fst->mtx);
+    // pthread_mutex_unlock(&fst->mtx);
     return false;
   } else {
     tOut = tOut + FST_NODE_FINAL_OUTPUT(root);
@@ -1018,8 +1073,8 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) {
     fstNodeDestroy(*node);
   }
   taosArrayDestroy(nodes);
-  fst->root = NULL;
-  pthread_mutex_unlock(&fst->mtx);
+  // fst->root = NULL;
+  // pthread_mutex_unlock(&fst->mtx);
   *out = tOut;
   return true;
 }
@@ -1028,7 +1083,9 @@ FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx) {
   return fstStreamBuilderCreate(fst, ctx);
 }
 StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb) {
-  if (sb == NULL) { return NULL; }
+  if (sb == NULL) {
+    return NULL;
+  }
   return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max);
 }
 FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) {
@@ -1039,15 +1096,6 @@ FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) {
 FstNode* fstGetRoot(Fst* fst) {
   CompiledAddr rAddr = fstGetRootAddr(fst);
   return fstGetNode(fst, rAddr);
-  // pthread_mutex_lock(&fst->mtx);
-  // if (fst->root != NULL) {
-  //  // pthread_mutex_unlock(&fst->mtx);
-  //  return fst->root;
-  //}
-  // CompiledAddr rAddr = fstGetRootAddr(fst);
-  // fst->root = fstGetNode(fst, rAddr);
-  //// pthread_mutex_unlock(&fst->mtx);
-  // return fst->root;
 }
 
 FstNode* fstGetNode(Fst* fst, CompiledAddr addr) {
@@ -1074,14 +1122,18 @@ bool fstVerify(Fst* fst) {
   uint32_t len, checkSum = fst->meta->checkSum;
   uint8_t* data = fstSliceData(fst->data, &len);
   TSCKSUM  initSum = 0;
-  if (!taosCheckChecksumWhole(data, len)) { return false; }
+  if (!taosCheckChecksumWhole(data, len)) {
+    return false;
+  }
   return true;
 }
 
 // data bound function
 FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data) {
   FstBoundWithData* b = calloc(1, sizeof(FstBoundWithData));
-  if (b == NULL) { return NULL; }
+  if (b == NULL) {
+    return NULL;
+  }
 
   if (data != NULL) {
     b->data = fstSliceCopy(data, data->start, data->end);
@@ -1118,7 +1170,9 @@ void fstBoundDestroy(FstBoundWithData* bound) { free(bound); }
 StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min,
                                        FstBoundWithData* max) {
   StreamWithState* sws = calloc(1, sizeof(StreamWithState));
-  if (sws == NULL) { return NULL; }
+  if (sws == NULL) {
+    return NULL;
+  }
 
   sws->fst = fst;
   sws->aut = automation;
@@ -1134,7 +1188,9 @@ StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstB
   return sws;
 }
 void streamWithStateDestroy(StreamWithState* sws) {
-  if (sws == NULL) { return; }
+  if (sws == NULL) {
+    return;
+  }
 
   taosArrayDestroy(sws->inp);
   taosArrayDestroyEx(sws->stack, streamStateDestroy);
@@ -1200,7 +1256,9 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) {
       uint64_t        i = 0;
       for (i = trans->range.start; i < trans->range.end; i++) {
         FstTransition trn;
-        if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) { break; }
+        if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) {
+          break;
+        }
       }
 
       StreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState};
@@ -1248,7 +1306,9 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
   while (taosArrayGetSize(sws->stack) > 0) {
     StreamState* p = (StreamState*)taosArrayPop(sws->stack);
     if (p->trans >= FST_NODE_LEN(p->node) || !automFuncs[aut->type].canMatch(aut, p->autState)) {
-      if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { taosArrayPop(sws->inp); }
+      if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) {
+        taosArrayPop(sws->inp);
+      }
       streamStateDestroy(p);
       continue;
     }
@@ -1267,7 +1327,9 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
     if (FST_NODE_IS_FINAL(nextNode)) {
       // void *eofState = sws->aut->acceptEof(nextState);
       void* eofState = automFuncs[aut->type].acceptEof(aut, nextState);
-      if (eofState != NULL) { isMatch = automFuncs[aut->type].isMatch(aut, eofState); }
+      if (eofState != NULL) {
+        isMatch = automFuncs[aut->type].isMatch(aut, eofState);
+      }
     }
     StreamState s1 = {.node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState};
     taosArrayPush(sws->stack, &s1);
@@ -1277,24 +1339,26 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
 
     size_t   isz = taosArrayGetSize(sws->inp);
     uint8_t* buf = (uint8_t*)malloc(isz * sizeof(uint8_t));
-    for (uint32_t i = 0; i < isz; i++) { buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); }
+    for (uint32_t i = 0; i < isz; i++) {
+      buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i);
+    }
     FstSlice slice = fstSliceCreate(buf, taosArrayGetSize(sws->inp));
     if (fstBoundWithDataExceededBy(sws->endAt, &slice)) {
       taosArrayDestroyEx(sws->stack, streamStateDestroy);
       sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState));
-      free(buf);
+      tfree(buf);
       fstSliceDestroy(&slice);
       return NULL;
     }
     if (FST_NODE_IS_FINAL(nextNode) && isMatch) {
       FstOutput              fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)};
       StreamWithStateResult* result = swsResultCreate(&slice, fOutput, tState);
-      free(buf);
+      tfree(buf);
       fstSliceDestroy(&slice);
       taosArrayDestroy(nodes);
       return result;
     }
-    free(buf);
+    tfree(buf);
     fstSliceDestroy(&slice);
   }
   for (size_t i = 0; i < taosArrayGetSize(nodes); i++) {
@@ -1307,16 +1371,19 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
 
 StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) {
   StreamWithStateResult* result = calloc(1, sizeof(StreamWithStateResult));
-  if (result == NULL) { return NULL; }
+  if (result == NULL) {
+    return NULL;
+  }
 
   result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1);
   result->out = fOut;
   result->state = state;
-
   return result;
 }
 void swsResultDestroy(StreamWithStateResult* result) {
-  if (NULL == result) { return; }
+  if (NULL == result) {
+    return;
+  }
 
   fstSliceDestroy(&result->data);
   startWithStateValueDestroy(result->state);
@@ -1324,16 +1391,18 @@ void swsResultDestroy(StreamWithStateResult* result) {
 }
 
 void streamStateDestroy(void* s) {
-  if (NULL == s) { return; }
+  if (NULL == s) {
+    return;
+  }
   StreamState* ss = (StreamState*)s;
-
   fstNodeDestroy(ss->node);
-  // free(s->autoState);
 }
 
 FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) {
   FstStreamBuilder* b = calloc(1, sizeof(FstStreamBuilder));
-  if (NULL == b) { return NULL; }
+  if (NULL == b) {
+    return NULL;
+  }
 
   b->fst = fst;
   b->aut = aut;
@@ -1349,8 +1418,9 @@ void fstStreamBuilderDestroy(FstStreamBuilder* b) {
   free(b);
 }
 FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type) {
-  if (b == NULL) { return NULL; }
-
+  if (b == NULL) {
+    return NULL;
+  }
   if (type == GE) {
     b->min->type = Included;
     fstSliceDestroy(&(b->min->data));

source/libs/index/src/index_fst_automation.c

@@ -17,7 +17,9 @@
 
 StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) {
   StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue));
-  if (nsv == NULL) { return NULL; }
+  if (nsv == NULL) {
+    return NULL;
+  }
 
   nsv->kind = kind;
   nsv->type = ty;
@@ -35,7 +37,9 @@ StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueTyp
 }
 void startWithStateValueDestroy(void* val) {
   StartWithStateValue* sv = (StartWithStateValue*)val;
-  if (sv == NULL) { return; }
+  if (sv == NULL) {
+    return;
+  }
 
   if (sv->type == FST_INT) {
     //
@@ -48,7 +52,9 @@ void startWithStateValueDestroy(void* val) {
 }
 StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) {
   StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue));
-  if (nsv == NULL) { return NULL; }
+  if (nsv == NULL) {
+    return NULL;
+  }
 
   nsv->kind = sv->kind;
   nsv->type = sv->type;
@@ -88,10 +94,14 @@ static bool prefixCanMatch(AutomationCtx* ctx, void* sv) {
 static bool  prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; }
 static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) {
   StartWithStateValue* ssv = (StartWithStateValue*)state;
-  if (ssv == NULL || ctx == NULL) { return NULL; }
+  if (ssv == NULL || ctx == NULL) {
+    return NULL;
+  }
 
   char* data = ctx->data;
-  if (ssv->kind == Done) { return startWithStateValueCreate(Done, FST_INT, &ssv->val); }
+  if (ssv->kind == Done) {
+    return startWithStateValueCreate(Done, FST_INT, &ssv->val);
+  }
   if ((strlen(data) > ssv->val) && data[ssv->val] == byte) {
     int val = ssv->val + 1;
 
@@ -128,7 +138,9 @@ AutomationFunc automFuncs[] = {
 
 AutomationCtx* automCtxCreate(void* data, AutomationType atype) {
   AutomationCtx* ctx = calloc(1, sizeof(AutomationCtx));
-  if (ctx == NULL) { return NULL; }
+  if (ctx == NULL) {
+    return NULL;
+  }
 
   StartWithStateValue* sv = NULL;
   if (atype == AUTOMATION_ALWAYS) {

source/libs/index/src/index_fst_util.c

@@ -29,18 +29,6 @@ const uint64_t VERSION = 3;
 
 const uint64_t TRANS_INDEX_THRESHOLD = 32;
 
-// uint8_t commonInput(uint8_t idx) {
-//  if (idx == 0) { return -1; }
-//  else {
-//    return COMMON_INPUTS_INV[idx - 1];
-//  }
-//}
-//
-// uint8_t commonIdx(uint8_t v, uint8_t max) {
-//  uint8_t v = ((uint16_t)tCOMMON_INPUTS[v] + 1)%256;
-//  return v > max ? 0: v;
-//}
-
 uint8_t packSize(uint64_t n) {
   if (n < (1u << 8)) {
     return 1;
@@ -103,9 +91,6 @@ FstSlice fstSliceCreate(uint8_t* data, uint64_t len) {
 FstSlice fstSliceCopy(FstSlice* s, int32_t start, int32_t end) {
   FstString* str = s->str;
   str->ref++;
-  // uint8_t *buf = fstSliceData(s, &alen);
-  // start = buf + start - (buf - s->start);
-  // end   = buf + end - (buf - s->start);
 
   FstSlice t = {.str = str, .start = start + s->start, .end = end + s->start};
   return t;
@@ -130,19 +115,19 @@ FstSlice fstSliceDeepCopy(FstSlice* s, int32_t start, int32_t end) {
   ans.end = tlen - 1;
   return ans;
 }
-bool fstSliceIsEmpty(FstSlice* s) {
-  return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0;
-}
+bool fstSliceIsEmpty(FstSlice* s) { return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0; }
 
 uint8_t* fstSliceData(FstSlice* s, int32_t* size) {
   FstString* str = s->str;
-  if (size != NULL) { *size = s->end - s->start + 1; }
+  if (size != NULL) {
+    *size = s->end - s->start + 1;
+  }
   return str->data + s->start;
 }
 void fstSliceDestroy(FstSlice* s) {
   FstString* str = s->str;
   str->ref--;
-  if (str->ref <= 0) {
+  if (str->ref == 0) {
     free(str->data);
     free(str);
     s->str = NULL;

source/libs/index/src/index_tfile.c

@@ -13,8 +13,6 @@ p *
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-//#include <sys/types.h>
-//#include <dirent.h>
 #include "index_tfile.h"
 #include "index.h"
 #include "index_fst.h"
@@ -61,7 +59,9 @@ static void    tfileGenFileFullName(char* fullname, const char* path, uint64_t s
 
 TFileCache* tfileCacheCreate(const char* path) {
   TFileCache* tcache = calloc(1, sizeof(TFileCache));
-  if (tcache == NULL) { return NULL; }
+  if (tcache == NULL) {
+    return NULL;
+  }
 
   tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
   tcache->capacity = 64;
@@ -98,7 +98,9 @@ End:
   return NULL;
 }
 void tfileCacheDestroy(TFileCache* tcache) {
-  if (tcache == NULL) { return; }
+  if (tcache == NULL) {
+    return;
+  }
 
   // free table cache
   TFileReader** reader = taosHashIterate(tcache->tableCache, NULL);
@@ -119,7 +121,9 @@ TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) {
   int32_t sz = indexSerialCacheKey(key, buf);
   assert(sz < sizeof(buf));
   TFileReader** reader = taosHashGet(tcache->tableCache, buf, sz);
-  if (reader == NULL) { return NULL; }
+  if (reader == NULL) {
+    return NULL;
+  }
   tfileReaderRef(*reader);
 
   return *reader;
@@ -142,7 +146,9 @@ void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) {
 }
 TFileReader* tfileReaderCreate(WriterCtx* ctx) {
   TFileReader* reader = calloc(1, sizeof(TFileReader));
-  if (reader == NULL) { return NULL; }
+  if (reader == NULL) {
+    return NULL;
+  }
 
   reader->ctx = ctx;
 
@@ -169,7 +175,9 @@ TFileReader* tfileReaderCreate(WriterCtx* ctx) {
   return reader;
 }
 void tfileReaderDestroy(TFileReader* reader) {
-  if (reader == NULL) { return; }
+  if (reader == NULL) {
+    return;
+  }
   // T_REF_INC(reader);
   fstDestroy(reader->fst);
   writerCtxDestroy(reader->ctx, reader->remove);
@@ -209,7 +217,9 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const c
   tfileGenFileFullName(fullname, path, suid, colName, version);
   // indexInfo("open write file name %s", fullname);
   WriterCtx* wcx = writerCtxCreate(TFile, fullname, false, 1024 * 1024 * 64);
-  if (wcx == NULL) { return NULL; }
+  if (wcx == NULL) {
+    return NULL;
+  }
 
   TFileHeader tfh = {0};
   tfh.suid = suid;
@@ -225,7 +235,9 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const c
 
   WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024);
   indexInfo("open read file name:%s, size: %d", wc->file.buf, wc->file.size);
-  if (wc == NULL) { return NULL; }
+  if (wc == NULL) {
+    return NULL;
+  }
 
   TFileReader* reader = tfileReaderCreate(wc);
   return reader;
@@ -316,19 +328,25 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) {
   return 0;
 }
 void tfileWriterClose(TFileWriter* tw) {
-  if (tw == NULL) { return; }
+  if (tw == NULL) {
+    return;
+  }
   writerCtxDestroy(tw->ctx, false);
   free(tw);
 }
 void tfileWriterDestroy(TFileWriter* tw) {
-  if (tw == NULL) { return; }
+  if (tw == NULL) {
+    return;
+  }
   writerCtxDestroy(tw->ctx, false);
   free(tw);
 }
 
 IndexTFile* indexTFileCreate(const char* path) {
   TFileCache* cache = tfileCacheCreate(path);
-  if (cache == NULL) { return NULL; }
+  if (cache == NULL) {
+    return NULL;
+  }
 
   IndexTFile* tfile = calloc(1, sizeof(IndexTFile));
   if (tfile == NULL) {
@@ -340,21 +358,27 @@ IndexTFile* indexTFileCreate(const char* path) {
   return tfile;
 }
 void indexTFileDestroy(IndexTFile* tfile) {
-  if (tfile == NULL) { return; }
+  if (tfile == NULL) {
+    return;
+  }
   tfileCacheDestroy(tfile->cache);
   free(tfile);
 }
 
 int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) {
   int ret = -1;
-  if (tfile == NULL) { return ret; }
+  if (tfile == NULL) {
+    return ret;
+  }
 
   IndexTFile* pTfile = (IndexTFile*)tfile;
 
   SIndexTerm* term = query->term;
   ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName};
   TFileReader* reader = tfileCacheGet(pTfile->cache, &key);
-  if (reader == NULL) { return 0; }
+  if (reader == NULL) {
+    return 0;
+  }
 
   return tfileReaderSearch(reader, query, result);
 }
@@ -373,7 +397,9 @@ static bool tfileIteratorNext(Iterate* iiter) {
 
   TFileFstIter*          tIter = iiter->iter;
   StreamWithStateResult* rt = streamWithStateNextWith(tIter->st, NULL);
-  if (rt == NULL) { return false; }
+  if (rt == NULL) {
+    return false;
+  }
 
   int32_t sz = 0;
   char*   ch = (char*)fstSliceData(&rt->data, &sz);
@@ -383,7 +409,9 @@ static bool tfileIteratorNext(Iterate* iiter) {
   offset = (uint64_t)(rt->out.out);
   swsResultDestroy(rt);
   // set up iterate value
-  if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { return false; }
+  if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) {
+    return false;
+  }
 
   iv->colVal = colVal;
   return true;
@@ -394,7 +422,9 @@ static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; }
 
 static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) {
   TFileFstIter* tIter = calloc(1, sizeof(TFileFstIter));
-  if (tIter == NULL) { return NULL; }
+  if (tIter == NULL) {
+    return NULL;
+  }
 
   tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS);
   tIter->fb = fstSearch(reader->fst, tIter->ctx);
@@ -404,7 +434,9 @@ static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) {
 }
 
 Iterate* tfileIteratorCreate(TFileReader* reader) {
-  if (reader == NULL) { return NULL; }
+  if (reader == NULL) {
+    return NULL;
+  }
 
   Iterate* iter = calloc(1, sizeof(Iterate));
   iter->iter = tfileFstIteratorCreate(reader);
@@ -419,7 +451,9 @@ Iterate* tfileIteratorCreate(TFileReader* reader) {
   return iter;
 }
 void tfileIteratorDestroy(Iterate* iter) {
-  if (iter == NULL) { return; }
+  if (iter == NULL) {
+    return;
+  }
 
   IterateValue* iv = &iter->val;
   iterateValueDestroy(iv, true);
@@ -434,7 +468,9 @@ void tfileIteratorDestroy(Iterate* iter) {
 }
 
 TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName) {
-  if (tf == NULL) { return NULL; }
+  if (tf == NULL) {
+    return NULL;
+  }
   ICacheKey key = {.suid = suid, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)};
   return tfileCacheGet(tf->cache, &key);
 }
@@ -446,7 +482,9 @@ static int tfileUidCompare(const void* a, const void* b) {
 }
 static int tfileStrCompare(const void* a, const void* b) {
   int ret = strcmp((char*)a, (char*)b);
-  if (ret == 0) { return ret; }
+  if (ret == 0) {
+    return ret;
+  }
   return ret < 0 ? -1 : 1;
 }
 
@@ -461,13 +499,17 @@ static int tfileValueCompare(const void* a, const void* b, const void* param) {
 
 TFileValue* tfileValueCreate(char* val) {
   TFileValue* tf = calloc(1, sizeof(TFileValue));
-  if (tf == NULL) { return NULL; }
+  if (tf == NULL) {
+    return NULL;
+  }
   tf->colVal = tstrdup(val);
   tf->tableId = taosArrayInit(32, sizeof(uint64_t));
   return tf;
 }
 int tfileValuePush(TFileValue* tf, uint64_t val) {
-  if (tf == NULL) { return -1; }
+  if (tf == NULL) {
+    return -1;
+  }
   taosArrayPush(tf->tableId, &val);
   return 0;
 }
@@ -489,7 +531,9 @@ static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset) {
   int32_t fstOffset = offset + sizeof(tw->header.fstOffset);
   tw->header.fstOffset = fstOffset;
 
-  if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { return -1; }
+  if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) {
+    return -1;
+  }
   indexInfo("tfile write fst offset: %d", tw->ctx->size(tw->ctx));
   tw->offset += sizeof(fstOffset);
   return 0;
@@ -502,7 +546,9 @@ static int tfileWriteHeader(TFileWriter* writer) {
 
   indexInfo("tfile pre write header size: %d", writer->ctx->size(writer->ctx));
   int nwrite = writer->ctx->write(writer->ctx, buf, sizeof(buf));
-  if (sizeof(buf) != nwrite) { return -1; }
+  if (sizeof(buf) != nwrite) {
+    return -1;
+  }
 
   indexInfo("tfile after write header size: %d", writer->ctx->size(writer->ctx));
   writer->offset = nwrite;
@@ -552,23 +598,23 @@ static int tfileReaderLoadHeader(TFileReader* reader) {
   return 0;
 }
 static int tfileReaderLoadFst(TFileReader* reader) {
-  // current load fst into memory, refactor it later
-  static int FST_MAX_SIZE = 64 * 1024 * 1024;
-
-  char* buf = calloc(1, sizeof(char) * FST_MAX_SIZE);
-  if (buf == NULL) { return -1; }
-
   WriterCtx* ctx = reader->ctx;
   int        size = ctx->size(ctx);
 
+  // current load fst into memory, refactor it later
+  int   fstSize = size - reader->header.fstOffset - sizeof(tfileMagicNumber);
+  char* buf = calloc(1, fstSize);
+  if (buf == NULL) {
+    return -1;
+  }
+
   int64_t ts = taosGetTimestampUs();
-  int32_t nread =
-      ctx->readFrom(ctx, buf, size - reader->header.fstOffset - sizeof(tfileMagicNumber), reader->header.fstOffset);
+  int32_t nread = ctx->readFrom(ctx, buf, fstSize, reader->header.fstOffset);
   int64_t cost = taosGetTimestampUs() - ts;
-  indexInfo("nread = %d, and fst offset=%d, filename: %s, size: %d, time cost: %" PRId64 "us", nread,
-            reader->header.fstOffset, ctx->file.buf, ctx->file.size, cost);
+  indexInfo("nread = %d, and fst offset=%d, size: %d, filename: %s, size: %d, time cost: %" PRId64 "us", nread,
+            reader->header.fstOffset, fstSize, ctx->file.buf, ctx->file.size, cost);
   // we assuse fst size less than FST_MAX_SIZE
-  assert(nread > 0 && nread < FST_MAX_SIZE);
+  assert(nread > 0 && nread <= fstSize);
 
   FstSlice st = fstSliceCreate((uint8_t*)buf, nread);
   reader->fst = fstCreate(&st);
@@ -578,21 +624,35 @@ static int tfileReaderLoadFst(TFileReader* reader) {
   return reader->fst != NULL ? 0 : -1;
 }
 static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result) {
-  int32_t    nid;
+  // TODO(yihao): opt later
   WriterCtx* ctx = reader->ctx;
+  char       block[1024] = {0};
+  int32_t    nread = ctx->readFrom(ctx, block, sizeof(block), offset);
+  assert(nread >= sizeof(uint32_t));
+
+  char*   p = block;
+  int32_t nid = *(int32_t*)p;
+  p += sizeof(nid);
+
+  while (nid > 0) {
+    int32_t left = block + sizeof(block) - p;
+    if (left >= sizeof(uint64_t)) {
+      taosArrayPush(result, (uint64_t*)p);
+      p += sizeof(uint64_t);
+    } else {
+      char buf[sizeof(uint64_t)] = {0};
+      memcpy(buf, p, left);
 
-  int32_t nread = ctx->readFrom(ctx, (char*)&nid, sizeof(nid), offset);
-  assert(sizeof(nid) == nread);
-
-  int32_t total = sizeof(uint64_t) * nid;
-  char*   buf = calloc(1, total);
-  if (buf == NULL) { return -1; }
-
-  nread = ctx->readFrom(ctx, buf, total, offset + sizeof(nid));
-  assert(total == nread);
+      memset(block, 0, sizeof(block));
+      offset += sizeof(block);
+      nread = ctx->readFrom(ctx, block, sizeof(block), offset);
+      memcpy(buf + left, block, sizeof(uint64_t) - left);
 
-  for (int32_t i = 0; i < nid; i++) { taosArrayPush(result, (uint64_t*)buf + i); }
-  free(buf);
+      taosArrayPush(result, (uint64_t*)buf);
+      p = block + sizeof(uint64_t) - left;
+    }
+    nid -= 1;
+  }
   return 0;
 }
 static int tfileReaderVerify(TFileReader* reader) {
@@ -615,13 +675,17 @@ static int tfileReaderVerify(TFileReader* reader) {
 }
 
 void tfileReaderRef(TFileReader* reader) {
-  if (reader == NULL) { return; }
+  if (reader == NULL) {
+    return;
+  }
   int ref = T_REF_INC(reader);
   UNUSED(ref);
 }
 
 void tfileReaderUnRef(TFileReader* reader) {
-  if (reader == NULL) { return; }
+  if (reader == NULL) {
+    return;
+  }
   int ref = T_REF_DEC(reader);
   if (ref == 0) {
     // do nothing
@@ -630,18 +694,21 @@ void tfileReaderUnRef(TFileReader* reader) {
 }
 
 static SArray* tfileGetFileList(const char* path) {
-  SArray* files = taosArrayInit(4, sizeof(void*));
-
   char     buf[128] = {0};
   uint64_t suid;
   uint32_t version;
+  SArray*  files = taosArrayInit(4, sizeof(void*));
 
   DIR* dir = opendir(path);
-  if (NULL == dir) { return NULL; }
+  if (NULL == dir) {
+    return NULL;
+  }
   struct dirent* entry;
   while ((entry = readdir(dir)) != NULL) {
     char* file = entry->d_name;
-    if (0 != tfileParseFileName(file, &suid, buf, &version)) { continue; }
+    if (0 != tfileParseFileName(file, &suid, buf, &version)) {
+      continue;
+    }
 
     size_t len = strlen(path) + 1 + strlen(file) + 1;
     char*  buf = calloc(1, len);

source/libs/index/test/CMakeLists.txt

 add_executable(indexTest "")
 add_executable(fstTest "")
+add_executable(fstUT "")
+
 target_sources(indexTest
   PRIVATE 
   "indexTests.cc" 
@@ -8,6 +10,11 @@ target_sources(fstTest
   PRIVATE 
   "fstTest.cc" 
 )
+
+target_sources(fstUT
+  PRIVATE 
+  "fstUT.cc" 
+)
 target_include_directories ( indexTest
   PUBLIC
   "${CMAKE_SOURCE_DIR}/include/libs/index" 
@@ -18,6 +25,12 @@ target_include_directories ( fstTest
   "${CMAKE_SOURCE_DIR}/include/libs/index" 
   "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 ) 
+
+target_include_directories ( fstUT 
+  PUBLIC
+  "${CMAKE_SOURCE_DIR}/include/libs/index" 
+  "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+) 
 target_link_libraries (indexTest
   os  
   util
@@ -32,6 +45,13 @@ target_link_libraries (fstTest
   gtest_main
   index
 )
+target_link_libraries (fstUT
+  os  
+  util
+  common
+  gtest_main
+  index
+)
 
 
 #add_test(

source/libs/index/test/fstTest.cc

@@ -58,7 +58,9 @@ class FstReadMemory {
   bool init() {
     char* buf = (char*)calloc(1, sizeof(char) * _size);
     int   nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size);
-    if (nRead <= 0) { return false; }
+    if (nRead <= 0) {
+      return false;
+    }
     _size = nRead;
     _s = fstSliceCreate((uint8_t*)buf, _size);
     _fst = fstCreate(&_s);
@@ -97,7 +99,8 @@ class FstReadMemory {
       printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out));
       swsResultDestroy(rt);
     }
-    for (size_t i = 0; i < result.size(); i++) {}
+    for (size_t i = 0; i < result.size(); i++) {
+    }
     std::cout << std::endl;
     return true;
   }
@@ -173,7 +176,9 @@ void checkMillonWriteAndReadOfFst() {
   delete fw;
   FstReadMemory* fr = new FstReadMemory(1024 * 64 * 1024);
 
-  if (fr->init()) { printf("success to init fst read"); }
+  if (fr->init()) {
+    printf("success to init fst read");
+  }
 
   Performance_fstReadRecords(fr);
   tfCleanup();

source/libs/index/test/fstUT.cc

+
+#include <gtest/gtest.h>
+#include <algorithm>
+#include <iostream>
+#include <string>
+#include <thread>
+#include <vector>
+#include "index.h"
+#include "indexInt.h"
+#include "index_cache.h"
+#include "index_fst.h"
+#include "index_fst_counting_writer.h"
+#include "index_fst_util.h"
+#include "index_tfile.h"
+#include "tglobal.h"
+#include "tskiplist.h"
+#include "tutil.h"
+#include "ulog.h"
+
+static std::string dir = "/tmp/index";
+
+static char indexlog[PATH_MAX] = {0};
+static char tindex[PATH_MAX] = {0};
+static char tindexDir[PATH_MAX] = {0};
+
+static void EnvInit() {
+  tfInit();
+
+  std::string path = dir;
+  taosRemoveDir(path.c_str());
+  taosMkDir(path.c_str());
+  // init log file
+  snprintf(indexlog, PATH_MAX, "%s/tindex.idx", path.c_str());
+  if (taosInitLog(indexlog, tsNumOfLogLines, 1) != 0) {
+    printf("failed to init log");
+  }
+  // init index file
+  memset(tindex, 0, sizeof(tindex));
+  snprintf(tindex, PATH_MAX, "%s/tindex.idx", path.c_str());
+}
+static void EnvCleanup() {}
+class FstWriter {
+ public:
+  FstWriter() {
+    _wc = writerCtxCreate(TFile, tindex, false, 64 * 1024 * 1024);
+    _b = fstBuilderCreate(_wc, 0);
+  }
+  bool Put(const std::string& key, uint64_t val) {
+    // char buf[128] = {0};
+    // int  len = 0;
+    // taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len);
+    // FstSlice skey = fstSliceCreate((uint8_t*)buf, len);
+    FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size());
+    bool     ok = fstBuilderInsert(_b, skey, val);
+
+    fstSliceDestroy(&skey);
+    return ok;
+  }
+  ~FstWriter() {
+    fstBuilderFinish(_b);
+    fstBuilderDestroy(_b);
+
+    writerCtxDestroy(_wc, false);
+  }
+
+ private:
+  FstBuilder* _b;
+  WriterCtx*  _wc;
+};
+
+class FstReadMemory {
+ public:
+  FstReadMemory(size_t size) {
+    _wc = writerCtxCreate(TFile, tindex, true, 64 * 1024);
+    _w = fstCountingWriterCreate(_wc);
+    _size = size;
+    memset((void*)&_s, 0, sizeof(_s));
+  }
+  bool init() {
+    char* buf = (char*)calloc(1, sizeof(char) * _size);
+    int   nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size);
+    if (nRead <= 0) {
+      return false;
+    }
+    _size = nRead;
+    _s = fstSliceCreate((uint8_t*)buf, _size);
+    _fst = fstCreate(&_s);
+    free(buf);
+    return _fst != NULL;
+  }
+  bool Get(const std::string& key, uint64_t* val) {
+    // char buf[128] = {0};
+    // int  len = 0;
+    // taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len);
+    // FstSlice skey = fstSliceCreate((uint8_t*)buf, len);
+
+    FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size());
+    bool     ok = fstGet(_fst, &skey, val);
+    fstSliceDestroy(&skey);
+    return ok;
+  }
+  bool GetWithTimeCostUs(const std::string& key, uint64_t* val, uint64_t* elapse) {
+    int64_t s = taosGetTimestampUs();
+    bool    ok = this->Get(key, val);
+    int64_t e = taosGetTimestampUs();
+    *elapse = e - s;
+    return ok;
+  }
+  // add later
+  bool Search(AutomationCtx* ctx, std::vector<uint64_t>& result) {
+    FstStreamBuilder*      sb = fstSearch(_fst, ctx);
+    StreamWithState*       st = streamBuilderIntoStream(sb);
+    StreamWithStateResult* rt = NULL;
+    while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
+      // result.push_back((uint64_t)(rt->out.out));
+      FstSlice*   s = &rt->data;
+      int32_t     sz = 0;
+      char*       ch = (char*)fstSliceData(s, &sz);
+      std::string key(ch, sz);
+      printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out));
+      swsResultDestroy(rt);
+    }
+    for (size_t i = 0; i < result.size(); i++) {
+    }
+    std::cout << std::endl;
+    return true;
+  }
+  bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector<uint64_t>& result) {
+    int64_t s = taosGetTimestampUs();
+    bool    ok = this->Search(ctx, result);
+    int64_t e = taosGetTimestampUs();
+    return ok;
+  }
+
+  ~FstReadMemory() {
+    fstCountingWriterDestroy(_w);
+    fstDestroy(_fst);
+    fstSliceDestroy(&_s);
+    writerCtxDestroy(_wc, false);
+    tfCleanup();
+  }
+
+ private:
+  FstCountingWriter* _w;
+  Fst*               _fst;
+  FstSlice           _s;
+  WriterCtx*         _wc;
+  size_t             _size;
+};
+
+class FstWriterEnv : public ::testing::Test {
+ protected:
+  virtual void SetUp() { fw = new FstWriter(); }
+  virtual void TearDown() { delete fw; }
+  FstWriter*   fw = NULL;
+};
+
+class FstReadEnv : public ::testing::Test {
+ protected:
+  virtual void   SetUp() { fr = new FstReadMemory(1024); }
+  virtual void   TearDown() { delete fr; }
+  FstReadMemory* fr = NULL;
+};
+
+class TFst {
+ public:
+  void CreateWriter() { fw = new FstWriter; }
+  void ReCreateWriter() {
+    if (fw != NULL) delete fw;
+    fw = new FstWriter;
+  }
+  void DestroyWriter() {
+    if (fw != NULL) delete fw;
+  }
+  void CreateReader() {
+    fr = new FstReadMemory(1024);
+    fr->init();
+  }
+  void ReCreateReader() {
+    if (fr != NULL) delete fr;
+    fr = new FstReadMemory(1024);
+  }
+  void DestroyReader() {
+    delete fr;
+    fr = NULL;
+  }
+  bool Put(const std::string& k, uint64_t v) {
+    if (fw == NULL) {
+      return false;
+    }
+    return fw->Put(k, v);
+  }
+  bool Get(const std::string& k, uint64_t* v) {
+    if (fr == NULL) {
+      return false;
+    }
+    return fr->Get(k, v);
+  }
+
+ private:
+  FstWriter*     fw;
+  FstReadMemory* fr;
+};
+class FstEnv : public ::testing::Test {
+ protected:
+  virtual void SetUp() {
+    EnvInit();
+    fst = new TFst;
+  }
+  virtual void TearDown() { delete fst; }
+  TFst*        fst;
+};
+
+TEST_F(FstEnv, writeNormal) {
+  fst->CreateWriter();
+  std::string str("aa");
+  for (int i = 0; i < 10; i++) {
+    str[0] = 'a' + i;
+    str.resize(2);
+    assert(fst->Put(str, i) == true);
+  }
+  // order failed
+  assert(fst->Put("aa", 1) == false);
+
+  fst->DestroyWriter();
+
+  fst->CreateReader();
+  uint64_t val;
+  assert(fst->Get("a", &val) == false);
+  assert(fst->Get("aa", &val) == true);
+  assert(val == 0);
+}
+TEST_F(FstEnv, writeExcpet) {}

source/libs/nodes/src/nodesCloneFuncs.c

@@ -15,6 +15,7 @@
 
 #include "querynodes.h"
 #include "taos.h"
+#include "taoserror.h"
 
 #define COPY_SCALAR_FIELD(fldname) \
 	do { \
@@ -57,14 +58,13 @@ static void dataTypeCopy(const SDataType* pSrc, SDataType* pDst) {
 }
 
 static void exprNodeCopy(const SExprNode* pSrc, SExprNode* pDst) {
-  COPY_SCALAR_FIELD(type);
   dataTypeCopy(&pSrc->resType, &pDst->resType);
   COPY_CHAR_ARRAY_FIELD(aliasName);
   // COPY_NODE_LIST_FIELD(pAssociationList);
 }
 
 static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) {
-  exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst);
+  exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst);
   COPY_SCALAR_FIELD(colId);
   COPY_SCALAR_FIELD(colType);
   COPY_CHAR_ARRAY_FIELD(dbName);
@@ -76,7 +76,7 @@ static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) {
 }
 
 static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) {
-  exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst);
+  exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst);
   COPY_CHAR_POINT_FIELD(literal);
   COPY_SCALAR_FIELD(isDuration);
   switch (pSrc->node.resType.type) {
@@ -119,7 +119,7 @@ static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) {
 }
 
 static SNode* operatorNodeCopy(const SOperatorNode* pSrc, SOperatorNode* pDst) {
-  exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst);
+  exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst);
   COPY_SCALAR_FIELD(opType);
   COPY_NODE_FIELD(pLeft);
   COPY_NODE_FIELD(pRight);
@@ -127,14 +127,14 @@ static SNode* operatorNodeCopy(const SOperatorNode* pSrc, SOperatorNode* pDst) {
 }
 
 static SNode* logicConditionNodeCopy(const SLogicConditionNode* pSrc, SLogicConditionNode* pDst) {
-  exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst);
+  exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst);
   COPY_SCALAR_FIELD(condType);
   COPY_NODE_LIST_FIELD(pParameterList);
   return (SNode*)pDst;
 }
 
 static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) {
-  exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst);
+  exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst);
   COPY_CHAR_ARRAY_FIELD(functionName);
   COPY_SCALAR_FIELD(funcId);
   COPY_SCALAR_FIELD(funcType);
@@ -142,12 +142,34 @@ static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) {
   return (SNode*)pDst;
 }
 
+static SNode* columnRefNodeCopy(const SColumnRefNode* pSrc, SColumnRefNode* pDst) {
+  dataTypeCopy(&pSrc->dataType, &pDst->dataType);
+  COPY_SCALAR_FIELD(tupleId);
+  COPY_SCALAR_FIELD(slotId);
+  COPY_SCALAR_FIELD(columnId);
+  return (SNode*)pDst;
+}
+
+static SNode* targetNodeCopy(const STargetNode* pSrc, STargetNode* pDst) {
+  COPY_SCALAR_FIELD(tupleId);
+  COPY_SCALAR_FIELD(slotId);
+  COPY_NODE_FIELD(pExpr);
+  return (SNode*)pDst;
+}
+
+static SNode* groupingSetNodeCopy(const SGroupingSetNode* pSrc, SGroupingSetNode* pDst) {
+  COPY_SCALAR_FIELD(groupingSetType);
+  COPY_NODE_LIST_FIELD(pParameterList);
+  return (SNode*)pDst;
+}
+
 SNode* nodesCloneNode(const SNode* pNode) {
   if (NULL == pNode) {
     return NULL;
   }
   SNode* pDst = nodesMakeNode(nodeType(pNode));
   if (NULL == pDst) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
   switch (nodeType(pNode)) {
@@ -161,10 +183,16 @@ SNode* nodesCloneNode(const SNode* pNode) {
       return logicConditionNodeCopy((const SLogicConditionNode*)pNode, (SLogicConditionNode*)pDst);
     case QUERY_NODE_FUNCTION:
       return functionNodeCopy((const SFunctionNode*)pNode, (SFunctionNode*)pDst);
+    case QUERY_NODE_COLUMN_REF:
+      return columnRefNodeCopy((const SColumnRefNode*)pNode, (SColumnRefNode*)pDst);
+    case QUERY_NODE_TARGET:
+      return targetNodeCopy((const STargetNode*)pNode, (STargetNode*)pDst);
     case QUERY_NODE_REAL_TABLE:
     case QUERY_NODE_TEMP_TABLE:
     case QUERY_NODE_JOIN_TABLE:
+      break;
     case QUERY_NODE_GROUPING_SET:
+      return groupingSetNodeCopy((const SGroupingSetNode*)pNode, (SGroupingSetNode*)pDst);
     case QUERY_NODE_ORDER_BY_EXPR:
     case QUERY_NODE_LIMIT:
     default:
@@ -174,8 +202,13 @@ SNode* nodesCloneNode(const SNode* pNode) {
 }
 
 SNodeList* nodesCloneList(const SNodeList* pList) {
+  if (NULL == pList) {
+    return NULL;
+  }
+
   SNodeList* pDst = nodesMakeList();
   if (NULL == pDst) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
   SNode* pNode;

source/libs/parser/test/mockCatalog.cpp

@@ -29,9 +29,9 @@
 namespace {
 
 void generateTestT1(MockCatalogService* mcs) {
-  ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 3)
+  ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 4)
       .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(1).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP)
-      .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 20);
+      .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 20).addColumn("c3", TSDB_DATA_TYPE_BIGINT);
   builder.done();
 }
 

source/libs/parser/test/parserTests.cpp

@@ -593,7 +593,7 @@ TEST(testCase, function_Test6) {
   SExprInfo* p2 = (SExprInfo*) taosArrayGetP(pQueryInfo->exprList[1], 0);
   ASSERT_EQ(p2->pExpr->nodeType, TEXPR_BINARYEXPR_NODE);
 
-  ASSERT_EQ(p2->pExpr->_node.optr, TSDB_BINARY_OP_ADD);
+  ASSERT_EQ(p2->pExpr->_node.optr, OP_TYPE_ADD);
   ASSERT_EQ(p2->pExpr->_node.pLeft->nodeType, TEXPR_COL_NODE);
   ASSERT_EQ(p2->pExpr->_node.pRight->nodeType, TEXPR_COL_NODE);
 

source/libs/planner/inc/plannerImpl.h

@@ -24,6 +24,7 @@ extern "C" {
 #include "planner.h"
 
 int32_t createLogicPlan(SNode* pNode, SLogicNode** pLogicNode);
+int32_t createPhysiPlan(SLogicNode* pLogicNode, SPhysiNode** pPhyNode);
 
 #ifdef __cplusplus
 }

source/libs/scalar/CMakeLists.txt

+aux_source_directory(src SCALAR_SRC)
+
+add_library(scalar STATIC ${SCALAR_SRC})
+target_include_directories(
+    scalar
+    PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scalar"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+)
+
+target_link_libraries(scalar
+        PRIVATE os util common nodes function qcom
+        )
+
+if(${BUILD_TEST})
+    ADD_SUBDIRECTORY(test)
+endif(${BUILD_TEST})
\ No newline at end of file

source/libs/function/inc/tbinoperator.h → source/libs/scalar/inc/sclvector.h

@@ -20,11 +20,10 @@
 extern "C" {
 #endif
 
-#include "tscalarfunction.h"
+#include "sclfunc.h"
 
-typedef void (*_bin_scalar_fn_t)(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *output, int32_t order);
+typedef void (*_bin_scalar_fn_t)(SScalarParam* pLeft, SScalarParam* pRight, void *output, int32_t order);
 _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binOperator);
-bool isBinaryStringOp(int32_t op);
 
 #ifdef __cplusplus
 }

source/libs/scalar/test/CMakeLists.txt

+enable_testing()
+
+add_subdirectory(filter)
+add_subdirectory(scalar)