Merge branch '3.0' into feature/TD-11274-3.0

include/client/taos.h

@@ -103,10 +103,10 @@ typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code);
 
 typedef struct TAOS_MULTI_BIND {
   int       buffer_type;
-  void *    buffer;
+  void     *buffer;
   uintptr_t buffer_length;
-  int32_t * length;
-  char *    is_null;
+  int32_t  *length;
+  char     *is_null;
   int       num;
 } TAOS_MULTI_BIND;
 
@@ -130,7 +130,7 @@ DLL_EXPORT void       taos_cleanup(void);
 DLL_EXPORT int        taos_options(TSDB_OPTION option, const void *arg, ...);
 DLL_EXPORT setConfRet taos_set_config(const char *config);
 DLL_EXPORT int        taos_init(void);
-DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
+DLL_EXPORT TAOS      *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
 DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen,
                                 const char *db, int dbLen, uint16_t port);
 DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port);
@@ -147,17 +147,17 @@ DLL_EXPORT int        taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name
 DLL_EXPORT int        taos_stmt_get_tag_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields);
 DLL_EXPORT int        taos_stmt_get_col_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields);
 
-DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
-DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums);
-DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes);
-DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
-DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
-DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx);
-DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt);
-DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt);
+DLL_EXPORT int       taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
+DLL_EXPORT int       taos_stmt_num_params(TAOS_STMT *stmt, int *nums);
+DLL_EXPORT int       taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes);
+DLL_EXPORT int       taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
+DLL_EXPORT int       taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
+DLL_EXPORT int       taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx);
+DLL_EXPORT int       taos_stmt_add_batch(TAOS_STMT *stmt);
+DLL_EXPORT int       taos_stmt_execute(TAOS_STMT *stmt);
 DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt);
 DLL_EXPORT int       taos_stmt_close(TAOS_STMT *stmt);
-DLL_EXPORT char *    taos_stmt_errstr(TAOS_STMT *stmt);
+DLL_EXPORT char     *taos_stmt_errstr(TAOS_STMT *stmt);
 DLL_EXPORT int       taos_stmt_affected_rows(TAOS_STMT *stmt);
 DLL_EXPORT int       taos_stmt_affected_rows_once(TAOS_STMT *stmt);
 
@@ -179,11 +179,11 @@ DLL_EXPORT bool        taos_is_update_query(TAOS_RES *res);
 DLL_EXPORT int         taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows);
 DLL_EXPORT int         taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows);
 DLL_EXPORT int         taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData);
-DLL_EXPORT int *       taos_get_column_data_offset(TAOS_RES *res, int columnIndex);
+DLL_EXPORT int        *taos_get_column_data_offset(TAOS_RES *res, int columnIndex);
 DLL_EXPORT int         taos_validate_sql(TAOS *taos, const char *sql);
 DLL_EXPORT void        taos_reset_current_db(TAOS *taos);
 
-DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res);
+DLL_EXPORT int      *taos_fetch_lengths(TAOS_RES *res);
 DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res);
 
 DLL_EXPORT const char *taos_get_server_info(TAOS *taos);
@@ -204,17 +204,17 @@ DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub);
 DLL_EXPORT void      taos_unsubscribe(TAOS_SUB *tsub, int keepProgress);
 #endif
 
-DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList);
+DLL_EXPORT int       taos_load_table_info(TAOS *taos, const char *tableNameList);
 DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision);
 
 /* --------------------------TMQ INTERFACE------------------------------- */
 
-enum tmq_resp_err_t {
+enum {
   TMQ_RESP_ERR__FAIL = -1,
   TMQ_RESP_ERR__SUCCESS = 0,
 };
 
-typedef enum tmq_resp_err_t tmq_resp_err_t;
+typedef int32_t tmq_resp_err_t;
 
 typedef struct tmq_t                   tmq_t;
 typedef struct tmq_topic_vgroup_t      tmq_topic_vgroup_t;
@@ -229,7 +229,7 @@ DLL_EXPORT tmq_list_t *tmq_list_new();
 DLL_EXPORT int32_t     tmq_list_append(tmq_list_t *, const char *);
 DLL_EXPORT void        tmq_list_destroy(tmq_list_t *);
 DLL_EXPORT int32_t     tmq_list_get_size(const tmq_list_t *);
-DLL_EXPORT char **     tmq_list_to_c_array(const tmq_list_t *);
+DLL_EXPORT char      **tmq_list_to_c_array(const tmq_list_t *);
 
 DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
 
@@ -240,7 +240,7 @@ DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
 DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list);
 DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq);
 DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
-DLL_EXPORT TAOS_RES *     tmq_consumer_poll(tmq_t *tmq, int64_t timeout);
+DLL_EXPORT TAOS_RES      *tmq_consumer_poll(tmq_t *tmq, int64_t timeout);
 DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq);
 DLL_EXPORT tmq_resp_err_t tmq_commit_sync(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets);
 DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, tmq_commit_cb *cb, void *param);
@@ -260,7 +260,7 @@ enum tmq_conf_res_t {
 
 typedef enum tmq_conf_res_t tmq_conf_res_t;
 
-DLL_EXPORT tmq_conf_t *   tmq_conf_new();
+DLL_EXPORT tmq_conf_t    *tmq_conf_new();
 DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value);
 DLL_EXPORT void           tmq_conf_destroy(tmq_conf_t *conf);
 DLL_EXPORT void           tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param);

include/common/tdatablock.h

@@ -206,7 +206,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
 size_t blockDataGetSize(const SSDataBlock* pBlock);
 size_t blockDataGetRowSize(SSDataBlock* pBlock);
 double blockDataGetSerialRowSize(const SSDataBlock* pBlock);
-size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock);
+size_t blockDataGetSerialMetaSize(uint32_t numOfCols);
 
 int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo);
 int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
@@ -238,7 +238,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool
                             const char* stbFullName, int32_t vgId);
 
 static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) {
-  return blockDataGetSerialMetaSize(pBlock) + blockDataGetSize(pBlock);
+  return blockDataGetSerialMetaSize(pBlock->info.numOfCols) + blockDataGetSize(pBlock);
 }
 
 static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data,

include/common/tmsg.h

@@ -2500,6 +2500,7 @@ typedef struct {
   int64_t   sliding;
   int64_t   dstTbUid;  
   int32_t   dstVgId;  // for stream  
+  SEpSet    epSet;
   char*     expr;
 } STableIndexInfo;
 

include/libs/catalog/catalog.h

@@ -71,16 +71,21 @@ typedef struct SCatalogReq {
   bool    forceUpdate;    
 } SCatalogReq;
 
+typedef struct SMetaRes {
+  int32_t code;
+  void*   pRes;
+} SMetaRes;
+
 typedef struct SMetaData {
-  SArray    *pDbVgroup;   // SArray<SArray<SVgroupInfo>*>
-  SArray    *pDbCfg;      // SArray<SDbCfgInfo>
-  SArray    *pDbInfo;     // SArray<SDbInfo>
-  SArray    *pTableMeta;  // SArray<STableMeta*>
-  SArray    *pTableHash;  // SArray<SVgroupInfo>
-  SArray    *pUdfList;    // SArray<SFuncInfo>
-  SArray    *pIndex;      // SArray<SIndexInfo>
-  SArray    *pUser;       // SArray<bool>
-  SArray    *pQnodeList;  // SArray<SQueryNodeAddr>
+  SArray    *pDbVgroup;   // pRes = SArray<SVgroupInfo>*
+  SArray    *pDbCfg;      // pRes = SDbCfgInfo*
+  SArray    *pDbInfo;     // pRes = SDbInfo*
+  SArray    *pTableMeta;  // pRes = STableMeta*
+  SArray    *pTableHash;  // pRes = SVgroupInfo*
+  SArray    *pUdfList;    // pRes = SFuncInfo*
+  SArray    *pIndex;      // pRes = SIndexInfo*
+  SArray    *pUser;       // pRes = bool*
+  SArray    *pQnodeList;  // pRes = SQueryNodeAddr*
 } SMetaData;
 
 typedef struct SCatalogCfg {

include/libs/function/function.h

@@ -197,6 +197,7 @@ typedef struct SAggFunctionInfo {
 struct SScalarParam {
   SColumnInfoData *columnData;
   SHashObj        *pHashFilter;
+  int32_t          hashValueType;
   void            *param;  // other parameter, such as meta handle from vnode, to extract table name/tag value
   int32_t          numOfRows;
 };

include/libs/function/functionMgt.h

@@ -122,6 +122,14 @@ typedef enum EFunctionType {
   // internal function
   FUNCTION_TYPE_SELECT_VALUE,
 
+  // distributed splitting functions
+  FUNCTION_TYPE_APERCENTILE_PARTIAL,
+  FUNCTION_TYPE_APERCENTILE_MERGE,
+  FUNCTION_TYPE_SPREAD_PARTIAL,
+  FUNCTION_TYPE_SPREAD_MERGE,
+  FUNCTION_TYPE_HISTOGRAM_PARTIAL,
+  FUNCTION_TYPE_HISTOGRAM_MERGE,
+
   // user defined funcion
   FUNCTION_TYPE_UDF = 10000
 } EFunctionType;

include/libs/qcom/query.h

@@ -210,8 +210,8 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
 #define NEED_CLIENT_RM_TBLMETA_ERROR(_code)                                                   \
   ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST ||       \
    (_code) == TSDB_CODE_PAR_INVALID_COLUMNS_NUM || (_code) == TSDB_CODE_PAR_INVALID_COLUMN || \
-   (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code == TSDB_CODE_PAR_VALUE_TOO_LONG) || \
-   (_code == TSDB_CODE_PAR_INVALID_DROP_COL))
+   (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code) == TSDB_CODE_PAR_VALUE_TOO_LONG || \
+   (_code) == TSDB_CODE_PAR_INVALID_DROP_COL || ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID))
 #define NEED_CLIENT_REFRESH_VG_ERROR(_code) \
   ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID)
 #define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED)

include/libs/sync/syncTools.h

@@ -33,7 +33,7 @@ SSyncNode* syncNodeAcquire(int64_t rid);
 void       syncNodeRelease(SSyncNode* pNode);
 
 int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
-int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
+int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcHandleInfo* pInfo);
 void    syncSetMsgCb(int64_t rid, const SMsgCb* msgcb);
 char*   sync2SimpleStr(int64_t rid);
 

include/libs/transport/trpc.h

@@ -33,7 +33,10 @@ extern int32_t tsRpcHeadSize;
 typedef struct {
   uint32_t clientIp;
   uint16_t clientPort;
-  char     user[TSDB_USER_LEN];
+  union {
+    char    user[TSDB_USER_LEN];
+    int64_t applyIndex;
+  };
 } SRpcConnInfo;
 
 typedef struct SRpcHandleInfo {

include/util/talgo.h

@@ -65,7 +65,7 @@ void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __
  * @param flags
  * @return
  */
-void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size, __compar_fn_t fn, int32_t flags);
+void *taosbsearch(const void *key, const void *base, int32_t nmemb, int32_t size, __compar_fn_t compar, int32_t flags);
 
 /**
  * adjust heap
@@ -82,7 +82,7 @@ void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size
  * @return
  */
 void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar,
-                    __ext_compar_fn_t compar, char* buf, bool maxroot);
+                    __ext_compar_fn_t compar, char *buf, bool maxroot);
 
 /**
  * sort heap to make sure it is a max/min root heap
@@ -97,8 +97,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const
  * @param maxroot: if heap is max root heap
  * @return
  */
-void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar,
-                  bool maxroot);
+void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, bool maxroot);
 
 #ifdef __cplusplus
 }

include/util/taoserror.h

@@ -339,9 +339,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE       TAOS_DEF_ERROR_CODE(0, 0x060D)
 #define TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE    TAOS_DEF_ERROR_CODE(0, 0x060E)
 #define TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP       TAOS_DEF_ERROR_CODE(0, 0x060F)
-#define TSDB_CODE_TDB_INVALID_ACTION            TAOS_DEF_ERROR_CODE(0, 0x0600)
-#define TSDB_CODE_TDB_INVALID_CREATE_TB_MSG     TAOS_DEF_ERROR_CODE(0, 0x0601)
-#define TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM      TAOS_DEF_ERROR_CODE(0, 0x0602)
+#define TSDB_CODE_TDB_INVALID_ACTION            TAOS_DEF_ERROR_CODE(0, 0x0610)
+#define TSDB_CODE_TDB_INVALID_CREATE_TB_MSG     TAOS_DEF_ERROR_CODE(0, 0x0611)
+#define TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM      TAOS_DEF_ERROR_CODE(0, 0x0612)
 #define TSDB_CODE_TDB_FILE_ALREADY_EXISTS       TAOS_DEF_ERROR_CODE(0, 0x0613)
 #define TSDB_CODE_TDB_TABLE_RECONFIGURE         TAOS_DEF_ERROR_CODE(0, 0x0614)
 #define TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO     TAOS_DEF_ERROR_CODE(0, 0x0615)

include/util/tdef.h

@@ -32,7 +32,7 @@ extern "C" {
 #define TD_VER_MAX UINT64_MAX  // TODO: use the real max version from query handle
 
 // Bytes for each type.
-extern const int32_t TYPE_BYTES[15];
+extern const int32_t TYPE_BYTES[16];
 
 // TODO: replace and remove code below
 #define CHAR_BYTES      sizeof(char)
@@ -356,8 +356,8 @@ typedef enum ELogicConditionType {
 
 #define TSDB_DEFAULT_EXPLAIN_VERBOSE false
 
-#define TSDB_EXPLAIN_RESULT_ROW_SIZE    1024
-#define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY PLAN"
+#define TSDB_EXPLAIN_RESULT_ROW_SIZE    512
+#define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY_PLAN"
 
 #define TSDB_MAX_FIELD_LEN             16384
 #define TSDB_MAX_BINARY_LEN            (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE)  // keep 16384

source/client/inc/clientInt.h

@@ -45,7 +45,7 @@ extern "C" {
 
 #define ERROR_MSG_BUF_DEFAULT_SIZE 512
 #define HEARTBEAT_INTERVAL         1500  // ms
-#define SYNC_ON_TOP_OF_ASYNC       0
+#define SYNC_ON_TOP_OF_ASYNC       1
 
 enum {
   RES_TYPE__QUERY = 1,
@@ -144,7 +144,7 @@ typedef struct STscObj {
   int32_t       numOfReqs;  // number of sqlObj bound to this connection
   SAppInstInfo* pAppInfo;
   SHashObj*     pRequests;
-  int8_t        schemalessType;
+  int8_t        schemalessType;  // todo remove it, this attribute should be move to request
 } STscObj;
 
 typedef struct SResultColumn {

source/client/src/clientImpl.c

@@ -580,7 +580,8 @@ void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) {
 
   STscObj* pTscObj = pRequest->pTscObj;
   if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code)) {
-    tscDebug("0x%"PRIx64" client retry to handle the error, code:%s, reqId:0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId);
+    tscDebug("0x%"PRIx64" client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%"PRIx64, pRequest->self, code, tstrerror(code),
+        pRequest->retry, pRequest->requestId);
     pRequest->prevCode = code;
     doAsyncQuery(pRequest, true);
     return;
@@ -592,6 +593,7 @@ void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) {
     pRequest->code = code;
   }
 
+  tscDebug("schedulerExecCb request type %s", TMSG_INFO(pRequest->type));
   if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) {
     removeMeta(pTscObj, pRequest->tableList);
   }
@@ -695,6 +697,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) {
       if (TSDB_CODE_SUCCESS == code) {
         schedulerAsyncExecJob(pAppInfo->pTransporter, pNodeList, pRequest->body.pDag, &pRequest->body.queryJob,
                               pRequest->sqlstr, pRequest->metric.start, schedulerExecCb, pRequest);
+      } else {
+        pRequest->body.queryFp(pRequest->body.param, pRequest, code);
       }
 
       //todo not to be released here
@@ -702,7 +706,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) {
       break;
     }
     case QUERY_EXEC_MODE_EMPTY_RESULT:
-      pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
+      pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT;      
+      pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
       break;
     default:
       break;
@@ -1280,8 +1285,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
             int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData),
                                            varDataVal(dst) + CHAR_BYTES);
             if (length <= 0) {
-              tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
-                       varDataVal(jsonInnerData));
+              tscError("charset:%s to %s. convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset);
               length = 0;
             }
             varDataSetLen(dst, length + CHAR_BYTES * 2);
@@ -1340,6 +1344,17 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
   uint64_t groupId = *(uint64_t*)p;
   p += sizeof(uint64_t);
 
+  // check fields
+  for(int32_t i = 0; i < numOfCols; ++i) {
+    int16_t type = *(int16_t*) p;
+    p += sizeof(int16_t);
+
+    int32_t bytes = *(int32_t*) p;
+    p += sizeof(int32_t);
+
+//    ASSERT(type == pFields[i].type && bytes == pFields[i].bytes);
+  }
+
   int32_t* colLength = (int32_t*)p;
   p += sizeof(int32_t) * numOfCols;
 

source/client/src/clientMain.c

@@ -500,7 +500,11 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
       return 0;
     }
 
-    doFetchRows(pRequest, false, true);
+#if SYNC_ON_TOP_OF_ASYNC
+    doAsyncFetchRow(pRequest, false, true);
+#else
+    doFetchRows(pRequest, true, true);
+#endif
 
     // TODO refactor
     SReqResultInfo *pResultInfo = &pRequest->body.resInfo;
@@ -625,8 +629,10 @@ void retrieveMetaCallback(SMetaData* pResultMeta, void* param, int32_t code) {
     taosMemoryFree(pWrapper);
     launchAsyncQuery(pRequest, pQuery);
   } else {
+    tscDebug("error happens, code:%d", code);
     if (NEED_CLIENT_HANDLE_ERROR(code)) {
-      tscDebug("0x%"PRIx64" client retry to handle the error, code:%s, reqId:0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId);
+      tscDebug("0x%"PRIx64" client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%"PRIx64, pRequest->self, code, tstrerror(code),
+               pRequest->retry, pRequest->requestId);
       pRequest->prevCode = code;
       doAsyncQuery(pRequest, true);
       return;
@@ -691,6 +697,7 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext** pCxt) {
                        .pTransporter = pTscObj->pAppInfo->pTransporter,
                        .pStmtCb = NULL,
                        .pUser = pTscObj->user,
+                       .schemalessType = pTscObj->schemalessType,
                        .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)),
                        .async = true,};
   return TSDB_CODE_SUCCESS;
@@ -699,13 +706,14 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext** pCxt) {
 void doAsyncQuery(SRequestObj* pRequest, bool updateMetaForce) {
   SParseContext* pCxt = NULL;
   STscObj *pTscObj = pRequest->pTscObj;
+  int32_t code = 0;
 
   if (pRequest->retry++ > REQUEST_TOTAL_EXEC_TIMES) {
-    pRequest->code = pRequest->prevCode;
+    code = pRequest->prevCode;
     goto _error;
   }
 
-  int32_t code = createParseContext(pRequest, &pCxt);
+  code = createParseContext(pRequest, &pCxt);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -742,7 +750,7 @@ void doAsyncQuery(SRequestObj* pRequest, bool updateMetaForce) {
   }
 
   _error:
-  tscError("0x%"PRIx64" error happens, code:%s, reqId:0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId);
+  tscError("0x%"PRIx64" error happens, code:%d - %s, reqId:0x%"PRIx64, pRequest->self, code, tstrerror(code), pRequest->requestId);
   terrno = code;
   pRequest->code = code;
   pRequest->body.queryFp(pRequest->body.param, pRequest, code);

source/client/src/tmq.c

@@ -323,7 +323,7 @@ static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) {
 
 int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) {
   SMqCommitCbParam* pParam = (SMqCommitCbParam*)param;
-  pParam->rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL;
+  pParam->rspErr = code;
   if (pParam->async) {
     if (pParam->automatic && pParam->tmq->commitCb) {
       pParam->tmq->commitCb(pParam->tmq, pParam->rspErr, (tmq_topic_vgroup_list_t*)pParam->offsets,
@@ -432,12 +432,13 @@ int32_t tmqCommitInner(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8_
     code = pParam->rspErr;
     tsem_destroy(&pParam->rspSem);
     taosMemoryFree(pParam);
+  } else {
+    code = 0;
   }
 
   // avoid double free if msg is sent
   buf = NULL;
 
-  code = 0;
 END:
   if (buf) taosMemoryFree(buf);
   /*if (pParam) taosMemoryFree(pParam);*/
@@ -445,9 +446,9 @@ END:
 
   if (code != 0 && async) {
     if (automatic) {
-      tmq->commitCb(tmq, TMQ_RESP_ERR__FAIL, (tmq_topic_vgroup_list_t*)pOffsets, tmq->commitCbUserParam);
+      tmq->commitCb(tmq, code, (tmq_topic_vgroup_list_t*)pOffsets, tmq->commitCbUserParam);
     } else {
-      userCb(tmq, TMQ_RESP_ERR__FAIL, (tmq_topic_vgroup_list_t*)pOffsets, userParam);
+      userCb(tmq, code, (tmq_topic_vgroup_list_t*)pOffsets, userParam);
     }
   }
 
@@ -1474,16 +1475,16 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
 tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) {
   if (tmq->status == TMQ_CONSUMER_STATUS__READY) {
     tmq_resp_err_t rsp = tmq_commit_sync(tmq, NULL);
-    if (rsp == TMQ_RESP_ERR__FAIL) {
-      return TMQ_RESP_ERR__FAIL;
+    if (rsp != TMQ_RESP_ERR__SUCCESS) {
+      return rsp;
     }
 
     tmq_list_t* lst = tmq_list_new();
     rsp = tmq_subscribe(tmq, lst);
     tmq_list_destroy(lst);
 
-    if (rsp == TMQ_RESP_ERR__FAIL) {
-      return TMQ_RESP_ERR__FAIL;
+    if (rsp != TMQ_RESP_ERR__SUCCESS) {
+      return rsp;
     }
   }
   // TODO: free resources
@@ -1493,8 +1494,11 @@ tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) {
 const char* tmq_err2str(tmq_resp_err_t err) {
   if (err == TMQ_RESP_ERR__SUCCESS) {
     return "success";
+  } else if (err == TMQ_RESP_ERR__FAIL) {
+    return "fail";
+  } else {
+    return tstrerror(err);
   }
-  return "fail";
 }
 
 const char* tmq_get_topic_name(TAOS_RES* res) {

source/client/test/clientTests.cpp

@@ -780,31 +780,28 @@ TEST(testCase, async_api_test) {
 
   taos_query(pConn, "use test");
 
-  TAOS_RES* pRes = taos_query(pConn, "select * from t1");
-
-  taos_query(pConn, "alter table t1 add column b int");
-  pRes = taos_query(pConn, "insert into t1 values(now, 1, 2)");
+  TAOS_RES* pRes = taos_query(pConn, "desc abc1.tu");
   if (taos_errno(pRes) != 0) {
     printf("failed, reason:%s\n", taos_errstr(pRes));
   }
 
-//  int32_t n = 0;
-//  TAOS_ROW    pRow = NULL;
-//  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
-//  int32_t     numOfFields = taos_num_fields(pRes);
-//
-//  char str[512] = {0};
-//  while ((pRow = taos_fetch_row(pRes)) != NULL) {
-//    int32_t* length = taos_fetch_lengths(pRes);
-//    for(int32_t i = 0; i < numOfFields; ++i) {
-//      printf("(%d):%d " , i, length[i]);
-//    }
-//    printf("\n");
-//
-//    int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
-//    printf("%s\n", str);
-//    memset(str, 0, sizeof(str));
-//  }
+  int32_t n = 0;
+  TAOS_ROW    pRow = NULL;
+  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
+  int32_t     numOfFields = taos_num_fields(pRes);
+
+  char str[512] = {0};
+  while ((pRow = taos_fetch_row(pRes)) != NULL) {
+    int32_t* length = taos_fetch_lengths(pRes);
+    for(int32_t i = 0; i < numOfFields; ++i) {
+      printf("(%d):%d " , i, length[i]);
+    }
+    printf("\n");
+
+    int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
+    printf("%s\n", str);
+    memset(str, 0, sizeof(str));
+  }
 
   taos_query_a(pConn, "alter table test.m1 comment 'abcde' ", queryCallback, pConn);
   getchar();

source/common/src/tdatablock.c

@@ -682,9 +682,9 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) {
  * @param pBlock
  * @return
  */
-size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) {
-  // | total rows/total length | block group id | each column length |
-  return sizeof(int32_t) + sizeof(uint64_t) + pBlock->info.numOfCols * sizeof(int32_t);
+size_t blockDataGetSerialMetaSize(uint32_t numOfCols) {
+  // | total rows/total length | block group id | column schema | each column length |
+  return sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)) + numOfCols * sizeof(int32_t);
 }
 
 double blockDataGetSerialRowSize(const SSDataBlock* pBlock) {
@@ -1217,6 +1217,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
   pBlock->info.numOfCols = numOfCols;
   pBlock->info.hasVarCol = pDataBlock->info.hasVarCol;
   pBlock->info.rowSize = pDataBlock->info.rowSize;
+  pBlock->info.groupId = pDataBlock->info.groupId;
 
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData  colInfo = {0};
@@ -1246,7 +1247,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
 }
 
 size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) {
-  int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(pBlock);
+  int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(pBlock->info.numOfCols);
 
   int32_t rowSize = pBlock->info.rowSize;
 
@@ -1271,12 +1272,12 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) {
 
 void colDataDestroy(SColumnInfoData* pColData) {
   if (IS_VAR_DATA_TYPE(pColData->info.type)) {
-    taosMemoryFree(pColData->varmeta.offset);
+    taosMemoryFreeClear(pColData->varmeta.offset);
   } else {
-    taosMemoryFree(pColData->nullbitmap);
+    taosMemoryFreeClear(pColData->nullbitmap);
   }
 
-  taosMemoryFree(pColData->pData);
+  taosMemoryFreeClear(pColData->pData);
 }
 
 static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) {
@@ -1885,34 +1886,44 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
 
 void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols,
                          int8_t needCompress) {
+  // todo extract method
   int32_t* actualLen = (int32_t*)data;
   data += sizeof(int32_t);
 
   uint64_t* groupId = (uint64_t*)data;
   data += sizeof(uint64_t);
 
+  for(int32_t i = 0; i < numOfCols; ++i) {
+    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
+    *((int16_t*) data) = pColInfoData->info.type;
+    data += sizeof(int16_t);
+
+    *((int32_t*) data) = pColInfoData->info.bytes;
+    data += sizeof(int32_t);
+  }
+
   int32_t* colSizes = (int32_t*)data;
   data += numOfCols * sizeof(int32_t);
 
-  *dataLen = (numOfCols * sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t));
+  *dataLen = blockDataGetSerialMetaSize(numOfCols);
 
   int32_t numOfRows = pBlock->info.rows;
   for (int32_t col = 0; col < numOfCols; ++col) {
     SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, col);
 
     // copy the null bitmap
+    size_t metaSize = 0;
     if (IS_VAR_DATA_TYPE(pColRes->info.type)) {
-      size_t metaSize = numOfRows * sizeof(int32_t);
+      metaSize = numOfRows * sizeof(int32_t);
       memcpy(data, pColRes->varmeta.offset, metaSize);
-      data += metaSize;
-      (*dataLen) += metaSize;
     } else {
-      int32_t len = BitmapLen(numOfRows);
-      memcpy(data, pColRes->nullbitmap, len);
-      data += len;
-      (*dataLen) += len;
+      metaSize = BitmapLen(numOfRows);
+      memcpy(data, pColRes->nullbitmap, metaSize);
     }
 
+    data += metaSize;
+    (*dataLen) += metaSize;
+
     if (needCompress) {
       colSizes[col] = blockCompressColData(pColRes, numOfRows, data, needCompress);
       data += colSizes[col];
@@ -1941,6 +1952,17 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t
   pBlock->info.groupId = *(uint64_t*)pStart;
   pStart += sizeof(uint64_t);
 
+  for(int32_t i = 0; i < numOfCols; ++i) {
+    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
+    pColInfoData->info.type = *(int16_t*)pStart;
+    pStart += sizeof(int16_t);
+
+    pColInfoData->info.bytes = *(int32_t*)pStart;
+    pStart += sizeof(int32_t);
+  }
+
+  blockDataEnsureCapacity(pBlock, numOfRows);
+
   int32_t* colLen = (int32_t*)pStart;
   pStart += sizeof(int32_t) * numOfCols;
 

source/common/src/tglobal.c

@@ -332,7 +332,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
   if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1;
   if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1;
-  if (cfgAddString(pCfg, "smlTagNullName", tsSmlTagName, 1) != 0) return -1;
+  if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1;
   if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1;
 
   tsNumOfTaskQueueThreads = tsNumOfCores / 4;
@@ -532,7 +532,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
   }
 
   tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN);
-  tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagNullName")->str, TSDB_COL_NAME_LEN);
+  tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN);
   tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval;
 
   tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32;

source/common/src/tmsg.c

@@ -2427,6 +2427,7 @@ int32_t tSerializeSTableIndexInfo(SEncoder *pEncoder, STableIndexInfo* pInfo) {
   if (tEncodeI64(pEncoder, pInfo->sliding) < 0) return -1;
   if (tEncodeI64(pEncoder, pInfo->dstTbUid) < 0) return -1;
   if (tEncodeI32(pEncoder, pInfo->dstVgId) < 0) return -1;
+  if (tEncodeSEpSet(pEncoder, &pInfo->epSet) < 0) return -1;
   if (tEncodeCStr(pEncoder, pInfo->expr) < 0) return -1;
   return 0;
 }
@@ -2459,6 +2460,7 @@ int32_t tDeserializeSTableIndexInfo(SDecoder *pDecoder, STableIndexInfo *pInfo)
   if (tDecodeI64(pDecoder, &pInfo->sliding) < 0) return -1;
   if (tDecodeI64(pDecoder, &pInfo->dstTbUid) < 0) return -1;
   if (tDecodeI32(pDecoder, &pInfo->dstVgId) < 0) return -1;
+  if (tDecodeSEpSet(pDecoder, &pInfo->epSet) < 0) return -1;
   if (tDecodeCStrAlloc(pDecoder, &pInfo->expr) < 0) return -1;
 
   return 0;

source/common/src/ttypes.c

@@ -18,7 +18,7 @@
 #include "tcompression.h"
 #include "trow.h"
 
-const int32_t TYPE_BYTES[15] = {
+const int32_t TYPE_BYTES[16] = {
     -1,                      // TSDB_DATA_TYPE_NULL
     CHAR_BYTES,              // TSDB_DATA_TYPE_BOOL
     CHAR_BYTES,              // TSDB_DATA_TYPE_TINYINT
@@ -34,6 +34,7 @@ const int32_t TYPE_BYTES[15] = {
     SHORT_BYTES,             // TSDB_DATA_TYPE_USMALLINT
     INT_BYTES,               // TSDB_DATA_TYPE_UINT
     sizeof(uint64_t),        // TSDB_DATA_TYPE_UBIGINT
+    TSDB_MAX_JSON_TAG_LEN,   // TSDB_DATA_TYPE_JSON
 };
 
 #define DO_STATICS(__sum, __min, __max, __minIndex, __maxIndex, _list, _index) \

source/dnode/mgmt/mgmt_vnode/inc/vmInt.h

@@ -34,6 +34,7 @@ typedef struct SVnodeMgmt {
   SQWorkerPool   fetchPool;
   SWWorkerPool   syncPool;
   SWWorkerPool   writePool;
+  SWWorkerPool   applyPool;
   SWWorkerPool   mergePool;
   SSingleWorker  mgmtWorker;
   SSingleWorker  monitorWorker;
@@ -52,10 +53,9 @@ typedef struct {
 
 typedef struct {
   int32_t     vgId;
-  int32_t     refCount;
   int32_t     vgVersion;
+  int32_t     refCount;
   int8_t      dropped;
-  int8_t      accessState;
   char       *path;
   SVnode     *pImpl;
   STaosQueue *pWriteQ;

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -323,7 +323,6 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_MON_VM_INFO, vmPutMsgToMonitorQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_MON_VM_LOAD, vmPutMsgToMonitorQueue, 0) == NULL) goto _OVER;
 
-  // Requests handled by VNODE
   if (dmSetMgmtHandle(pArray, TDMT_VND_SUBMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER;

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -49,10 +49,9 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) {
   }
 
   pVnode->vgId = pCfg->vgId;
-  pVnode->refCount = 0;
   pVnode->vgVersion = pCfg->vgVersion;
+  pVnode->refCount = 0;
   pVnode->dropped = 0;
-  pVnode->accessState = TSDB_VN_ALL_ACCCESS;
   pVnode->path = tstrdup(pCfg->path);
   pVnode->pImpl = pImpl;
 
@@ -96,7 +95,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
   dDebug("vgId:%d, vnode is closed", pVnode->vgId);
 
   if (pVnode->dropped) {
-    dDebug("vgId:%d, vnode is destroyed for dropped:%d", pVnode->vgId, pVnode->dropped);
+    dInfo("vgId:%d, vnode is destroyed, dropped:%d", pVnode->vgId, pVnode->dropped);
     snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pVnode->vgId);
     vnodeDestroy(path, pMgmt->pTfs);
   }

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -16,10 +16,8 @@
 #define _DEFAULT_SOURCE
 #include "vmInt.h"
 
-#include "sync.h"
-#include "syncTools.h"
-
 static inline void vmSendRsp(SRpcMsg *pMsg, int32_t code) {
+  if (pMsg->info.handle == NULL) return;
   SRpcMsg rsp = {
       .code = code,
       .pCont = pMsg->info.rsp,
@@ -53,9 +51,9 @@ static void vmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
   }
 
   if (IsReq(pMsg)) {
-    if (code != 0 && terrno != 0) {
-      dError("msg:%p failed to process since %s", pMsg, terrstr());
-      code = terrno;
+    if (code != 0) {
+      if (terrno != 0) code = terrno;
+      dError("msg:%p, failed to process since %s", pMsg, terrstr());
     }
     vmSendRsp(pMsg, code);
   }
@@ -97,110 +95,6 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
   taosFreeQitem(pMsg);
 }
 
-static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
-  int32_t    code = 0;
-  SRpcMsg   *pMsg = NULL;
-  SVnodeObj *pVnode = pInfo->ahandle;
-  int64_t    sync = vnodeGetSyncHandle(pVnode->pImpl);
-  SArray    *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg **));
-
-  for (int32_t m = 0; m < numOfMsgs; m++) {
-    if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
-    dTrace("vgId:%d, msg:%p get from vnode-write queue", pVnode->vgId, pMsg);
-
-    if (taosArrayPush(pArray, &pMsg) == NULL) {
-      dError("vgId:%d, failed to push msg:%p to vnode-write array", pVnode->vgId, pMsg);
-      vmSendRsp(pMsg, TSDB_CODE_OUT_OF_MEMORY);
-    }
-  }
-
-  for (int32_t m = 0; m < taosArrayGetSize(pArray); m++) {
-    pMsg = *(SRpcMsg **)taosArrayGet(pArray, m);
-    code = vnodePreprocessReq(pVnode->pImpl, pMsg);
-
-    if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
-      dTrace("vgId:%d, msg:%p in progress and no rsp", pVnode->vgId, pMsg);
-      continue;
-    }
-
-    if (pMsg->msgType != TDMT_VND_ALTER_REPLICA) {
-      code = syncPropose(sync, pMsg, false);
-    }
-
-    if (code == TAOS_SYNC_PROPOSE_SUCCESS) {
-      dTrace("vgId:%d, msg:%p is proposed and no rsp", pVnode->vgId, pMsg);
-      continue;
-    } else if (code == TAOS_SYNC_PROPOSE_NOT_LEADER) {
-      SEpSet newEpSet = {0};
-      syncGetEpSet(sync, &newEpSet);
-      SEp *pEp = &newEpSet.eps[newEpSet.inUse];
-      if (pEp->port == tsServerPort && strcmp(pEp->fqdn, tsLocalFqdn) == 0) {
-        newEpSet.inUse = (newEpSet.inUse + 1) % newEpSet.numOfEps;
-      }
-
-      dTrace("vgId:%d, msg:%p is redirect since not leader, numOfEps:%d inUse:%d", pVnode->vgId, pMsg,
-             newEpSet.numOfEps, newEpSet.inUse);
-      for (int32_t i = 0; i < newEpSet.numOfEps; ++i) {
-        dTrace("vgId:%d, msg:%p ep:%s:%u", pVnode->vgId, pMsg, newEpSet.eps[i].fqdn, newEpSet.eps[i].port);
-      }
-
-      SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info};
-      tmsgSendRedirectRsp(&rsp, &newEpSet);
-    } else {
-      dError("vgId:%d, msg:%p failed to propose write since %s, code:0x%x", pVnode->vgId, pMsg, tstrerror(code), code);
-      vmSendRsp(pMsg, code);
-    }
-  }
-
-  for (int32_t i = 0; i < numOfMsgs; i++) {
-    pMsg = *(SRpcMsg **)taosArrayGet(pArray, i);
-    dTrace("vgId:%d, msg:%p is freed", pVnode->vgId, pMsg);
-    rpcFreeCont(pMsg->pCont);
-    taosFreeQitem(pMsg);
-  }
-
-  taosArrayDestroy(pArray);
-}
-
-static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
-  SVnodeObj *pVnode = pInfo->ahandle;
-  SRpcMsg   *pMsg = NULL;
-
-  for (int32_t i = 0; i < numOfMsgs; ++i) {
-    if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
-    dTrace("vgId:%d, msg:%p get from vnode-apply queue", pVnode->vgId, pMsg);
-
-    // init response rpc msg
-    SRpcMsg rsp = {0};
-
-    // get original rpc msg
-    assert(pMsg->msgType == TDMT_SYNC_APPLY_MSG);
-    SyncApplyMsg *pSyncApplyMsg = syncApplyMsgFromRpcMsg2(pMsg);
-    syncApplyMsgLog2("==vmProcessApplyQueue==", pSyncApplyMsg);
-    SRpcMsg originalRpcMsg;
-    syncApplyMsg2OriginalRpcMsg(pSyncApplyMsg, &originalRpcMsg);
-
-    // apply data into tsdb
-    if (vnodeProcessWriteReq(pVnode->pImpl, &originalRpcMsg, pSyncApplyMsg->fsmMeta.index, &rsp) < 0) {
-      rsp.code = terrno;
-      dError("vgId:%d, msg:%p failed to apply since %s", pVnode->vgId, pMsg, terrstr());
-    }
-
-    syncApplyMsgDestroy(pSyncApplyMsg);
-    rpcFreeCont(originalRpcMsg.pCont);
-
-    // if leader, send response
-    if (pMsg->info.handle != NULL) {
-      rsp.info = pMsg->info;
-      tmsgSendRsp(&rsp);
-    }
-
-    dTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, rsp.code);
-    rpcFreeCont(pMsg->pCont);
-    taosFreeQitem(pMsg);
-  }
-}
-
 static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
   SVnodeObj *pVnode = pInfo->ahandle;
   SRpcMsg   *pMsg = NULL;
@@ -211,11 +105,9 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf
 
     int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL);
     if (code != 0) {
+      if (terrno != 0) code = terrno;
       dError("vgId:%d, msg:%p failed to sync since %s", pVnode->vgId, pMsg, terrstr());
-      if (pMsg->info.handle != NULL) {
-        if (terrno != 0) code = terrno;
-        vmSendRsp(pMsg, code);
-      }
+      vmSendRsp(pMsg, code);
     }
 
     dTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code);
@@ -234,8 +126,8 @@ static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
 
     int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo);
     if (code != 0) {
-      dError("vgId:%d, msg:%p failed to merge since %s", pVnode->vgId, pMsg, terrstr());
       if (terrno != 0) code = terrno;
+      dError("vgId:%d, msg:%p failed to merge since %s", pVnode->vgId, pMsg, terrstr());
       vmSendRsp(pMsg, code);
     }
 
@@ -254,7 +146,7 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
 
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
   if (pVnode == NULL) {
-    dError("vgId:%d, failed to put msg:%p into vnode queue since %s, type:%s", pHead->vgId, pMsg, terrstr(),
+    dError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s", pHead->vgId, pMsg, terrstr(),
            TMSG_INFO(pMsg->msgType));
     return terrno != 0 ? terrno : -1;
   }
@@ -322,7 +214,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
   if (pMsg == NULL) return -1;
 
   SMsgHead *pHead = pRpc->pCont;
-  dTrace("vgId:%d, msg:%p is created, type:%s", pHead->vgId, pMsg, TMSG_INFO(pMsg->msgType));
+  dTrace("vgId:%d, msg:%p is created, type:%s", pHead->vgId, pMsg, TMSG_INFO(pRpc->msgType));
 
   pHead->contLen = htonl(pHead->contLen);
   pHead->vgId = htonl(pHead->vgId);
@@ -362,9 +254,9 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
 }
 
 int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
-  pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessWriteQueue);
+  pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode->pImpl, (FItems)vnodeProposeMsg);
   pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue);
-  pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessApplyQueue);
+  pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->applyPool, pVnode->pImpl, (FItems)vnodeApplyMsg);
   pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue);
   pVnode->pFetchQ = tQWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)vmProcessFetchQueue);
   pVnode->pMergeQ = tWWorkerAllocQueue(&pMgmt->mergePool, pVnode, (FItems)vmProcessMergeQueue);
@@ -381,8 +273,8 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
 
 void vmFreeQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
   tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ);
+  tWWorkerFreeQueue(&pMgmt->applyPool, pVnode->pApplyQ);
   tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ);
-  tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ);
   tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ);
   tQWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ);
   tWWorkerFreeQueue(&pMgmt->mergePool, pVnode->pMergeQ);
@@ -413,6 +305,11 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) {
   pWPool->max = tsNumOfVnodeWriteThreads;
   if (tWWorkerInit(pWPool) != 0) return -1;
 
+  SWWorkerPool *pAPool = &pMgmt->applyPool;
+  pAPool->name = "vnode-apply";
+  pAPool->max = tsNumOfVnodeWriteThreads;
+  if (tWWorkerInit(pAPool) != 0) return -1;
+
   SWWorkerPool *pSPool = &pMgmt->syncPool;
   pSPool->name = "vnode-sync";
   pSPool->max = tsNumOfVnodeSyncThreads;
@@ -449,6 +346,7 @@ void vmStopWorker(SVnodeMgmt *pMgmt) {
   tSingleWorkerCleanup(&pMgmt->monitorWorker);
   tSingleWorkerCleanup(&pMgmt->mgmtWorker);
   tWWorkerCleanup(&pMgmt->writePool);
+  tWWorkerCleanup(&pMgmt->applyPool);
   tWWorkerCleanup(&pMgmt->syncPool);
   tQWorkerCleanup(&pMgmt->queryPool);
   tQWorkerCleanup(&pMgmt->fetchPool);

source/dnode/mnode/impl/src/mndConsumer.c

@@ -92,6 +92,15 @@ static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) {
   SMqConsumerObj     *pConsumer = mndAcquireConsumer(pMnode, pLostMsg->consumerId);
   ASSERT(pConsumer);
 
+
+  mInfo("receive consumer lost msg, consumer id %ld, status %s", pLostMsg->consumerId,
+        mndConsumerStatusName(pConsumer->status));
+
+  if (pConsumer->status != MQ_CONSUMER_STATUS__READY) {
+    mndReleaseConsumer(pMnode, pConsumer);
+    return -1;
+  }
+
   SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup);
   pConsumerNew->updateType = CONSUMER_UPDATE__LOST;
 

source/dnode/mnode/impl/src/mndShow.c

@@ -266,7 +266,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
   }
 
   size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pShow->pMeta->numOfColumns +
-         blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(pBlock);
+         blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(pBlock->info.numOfCols);
 
   SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size);
   if (pRsp == NULL) {

source/dnode/mnode/impl/src/mndSma.c

@@ -894,6 +894,15 @@ static int32_t mndGetTableSma(SMnode *pMnode, STableIndexReq *indexReq, STableIn
     info.sliding = pSma->sliding;
     info.dstTbUid = pSma->dstTbUid;
     info.dstVgId = pSma->dstVgId;
+
+    SVgObj* pVg = mndAcquireVgroup(pMnode, pSma->dstVgId);
+    if (pVg == NULL) {
+      code = -1;
+      sdbRelease(pSdb, pSma);
+      return code;
+    }
+    info.epSet = mndGetVgroupEpset(pMnode, pVg);
+    
     info.expr = taosMemoryMalloc(pSma->exprLen + 1);
     if (info.expr == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;

source/dnode/vnode/CMakeLists.txt

@@ -36,12 +36,10 @@ target_sources(
 
     # tsdb
     "src/tsdb/tsdbCommit.c"
-    # "src/tsdb/tsdbCommit2.c"
     "src/tsdb/tsdbFile.c"
     "src/tsdb/tsdbFS.c"
     "src/tsdb/tsdbOpen.c"
     "src/tsdb/tsdbMemTable.c"
-    # "src/tsdb/tsdbMemTable2.c"
     "src/tsdb/tsdbRead.c"
     "src/tsdb/tsdbReadImpl.c"
     "src/tsdb/tsdbWrite.c"

source/dnode/vnode/inc/vnode.h

@@ -71,6 +71,9 @@ int32_t vnodeSnapshotReaderClose(SVSnapshotReader *pReader);
 int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32_t *nData);
 int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen);
 
+void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs);
+void vnodeApplyMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs);
+
 // meta
 typedef struct SMeta       SMeta;  // todo: remove
 typedef struct SMetaReader SMetaReader;

source/dnode/vnode/src/inc/tsdb.h

@@ -58,28 +58,6 @@ void    tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, ST
 bool    tsdbTbDataIterNext(STbDataIter *pIter);
 bool    tsdbTbDataIterGet(STbDataIter *pIter, TSDBROW *pRow);
 
-int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, STbDataIter *pIter, TSKEY maxKey, int maxRowsToRead,
-                          SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo);
-
-// tsdbMemTable2.c ==============================================================================================
-// typedef struct SMemTable2   SMemTable2;
-// typedef struct SMemData     SMemData;
-// typedef struct SMemDataIter SMemDataIter;
-
-// int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable2 **ppMemTable);
-// void    tsdbMemTableDestroy2(SMemTable2 *pMemTable);
-// int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmitBlk);
-// int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
-
-// /* SMemDataIter */
-// void tsdbMemDataIterOpen(SMemData *pMemData, TSDBKEY *pKey, int8_t backward, SMemDataIter *pIter);
-// bool tsdbMemDataIterNext(SMemDataIter *pIter);
-// void tsdbMemDataIterGet(SMemDataIter *pIter, TSDBROW **ppRow);
-
-// // tsdbCommit2.c ==============================================================================================
-// int32_t tsdbBegin2(STsdb *pTsdb);
-// int32_t tsdbCommit2(STsdb *pTsdb);
-
 // tsdbFile.c ==============================================================================================
 typedef int32_t          TSDB_FILE_T;
 typedef struct SDFInfo   SDFInfo;
@@ -700,17 +678,6 @@ typedef struct {
   TSKEY    eKey;
 } SDelInfo;
 
-struct SMemTable2 {
-  STsdb  *pTsdb;
-  int32_t nRef;
-  TSDBKEY minKey;
-  TSDBKEY maxKey;
-  int64_t nRows;
-  int64_t nDelOp;
-  SArray *aSkmInfo;
-  SArray *aMemData;
-};
-
 static FORCE_INLINE int tsdbKeyCmprFn(const void *p1, const void *p2) {
   TSDBKEY *pKey1 = (TSDBKEY *)p1;
   TSDBKEY *pKey2 = (TSDBKEY *)p2;
@@ -730,24 +697,6 @@ static FORCE_INLINE int tsdbKeyCmprFn(const void *p1, const void *p2) {
   return 0;
 }
 
-struct SMemData {
-  tb_uid_t     suid;
-  tb_uid_t     uid;
-  TSDBKEY      minKey;
-  TSDBKEY      maxKey;
-  SDelOp      *delOpHead;
-  SDelOp      *delOpTail;
-  SMemSkipList sl;
-};
-
-struct SMemDataIter {
-  STbData          *pMemData;
-  int8_t            backward;
-  TSDBROW          *pRow;
-  SMemSkipListNode *pNode;  // current node
-  TSDBROW           row;
-};
-
 struct STbDataIter {
   STbData          *pTbData;
   int8_t            backward;

source/dnode/vnode/src/inc/vnd.h

@@ -84,7 +84,6 @@ int32_t vnodeAsyncCommit(SVnode* pVnode);
 int32_t vnodeSyncOpen(SVnode* pVnode, char* path);
 void    vnodeSyncStart(SVnode* pVnode);
 void    vnodeSyncClose(SVnode* pVnode);
-int32_t vnodeSyncAlter(SVnode* pVnode, SRpcMsg* pMsg);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/inc/vnodeInt.h

@@ -112,7 +112,7 @@ int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid);
 // tsdb
 int          tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg);
 int          tsdbClose(STsdb** pTsdb);
-int          tsdbBegin(STsdb* pTsdb);
+int32_t      tsdbBegin(STsdb* pTsdb);
 int32_t      tsdbCommit(STsdb* pTsdb);
 int          tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg);
 int          tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp);
@@ -161,18 +161,6 @@ int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore);
 void    tdUidStoreDestory(STbUidStore* pStore);
 void*   tdUidStoreFree(STbUidStore* pStore);
 
-#if 0
-int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version);
-int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg);
-int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg);
-int32_t tsdbRegisterRSma(STsdb* pTsdb, SMeta* pMeta, SVCreateStbReq* pReq, SMsgCb* pMsgCb);
-int32_t tsdbFetchTbUidList(STsdb* pTsdb, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid);
-int32_t tsdbUpdateTbUidList(STsdb* pTsdb, STbUidStore* pUidStore);
-void    tsdbUidStoreDestory(STbUidStore* pStore);
-void*   tsdbUidStoreFree(STbUidStore* pStore);
-int32_t tsdbTriggerRSma(STsdb* pTsdb, void* pMsg, int32_t inputType);
-#endif
-
 typedef struct {
   int8_t  streamType;  // sma or other
   int8_t  dstType;
@@ -232,8 +220,10 @@ struct SVnode {
   SWal*      pWal;
   STQ*       pTq;
   SSink*     pSink;
-  int64_t    sync;
   tsem_t     canCommit;
+  int64_t    sync;
+  int32_t    syncCount;
+  sem_t      syncSem;
   SQHandle*  pQuery;
 };
 

source/dnode/vnode/src/meta/metaTable.c

@@ -61,14 +61,14 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
   if (tTagToValArray((const STag *)data, &pTagVals) != 0) {
     return -1;
   }
-  char key[512] = {0};
 
   SIndexMultiTerm *terms = indexMultiTermCreate();
   int16_t          nCols = taosArrayGetSize(pTagVals);
   for (int i = 0; i < nCols; i++) {
     STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i);
     char     type = pTagVal->type;
-    sprintf(key, "%s_%s", tagName, pTagVal->pKey);
+
+    char *  key = pTagVal->pKey;
     int32_t nKey = strlen(key);
 
     SIndexTerm *term = NULL;
@@ -93,12 +93,11 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
     } else if (type == TSDB_DATA_TYPE_BOOL) {
       int val = *(int *)(&pTagVal->i64);
       int len = 0;
-      term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, (const char *)&val, len);
+      term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_INT, key, nKey, (const char *)&val, len);
     }
     if (term != NULL) {
       indexMultiTermAdd(terms, term);
     }
-    memset(key, 0, sizeof(key));
   }
   tIndexJsonPut(pMeta->pTagIvtIdx, terms, tuid);
   indexMultiTermDestroy(terms);

source/dnode/vnode/src/tq/tqExec.c

@@ -39,6 +39,9 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRs
 
 static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, int32_t workerId, SMqDataBlkRsp* pRsp) {
   SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pExecReader[workerId]->pSchemaWrapper);
+  if (pSW == NULL) {
+    return -1;
+  }
   taosArrayPush(pRsp->blockSchema, &pSW);
   return 0;
 }

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -28,6 +28,8 @@ typedef struct {
   int          niters;  // memory iterators
   SCommitIter *iters;
   bool         isRFileSet;  // read and commit FSET
+  int32_t      fid;
+  SDFileSet   *pSet;
   SReadH       readh;
   SDFileSet    wSet;
   bool         isDFileSame;
@@ -58,8 +60,12 @@ typedef struct {
 #define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->pVnode->config.tsdbCfg.maxRows)
 #define TSDB_COMMIT_TXN_VERSION(ch)  FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch)))
 
-static void    tsdbStartCommit(STsdb *pRepo);
-static void    tsdbEndCommit(STsdb *pTsdb, int eno);
+static int32_t tsdbCommitData(SCommitH *pCommith);
+static int32_t tsdbCommitDel(SCommitH *pCommith);
+static int32_t tsdbCommitCache(SCommitH *pCommith);
+static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitH *pCHandle);
+static int32_t tsdbEndCommit(SCommitH *pCHandle, int eno);
+
 static int     tsdbInitCommitH(SCommitH *pCommith, STsdb *pRepo);
 static void    tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key);
 static int     tsdbNextCommitFid(SCommitH *pCommith);
@@ -67,7 +73,6 @@ static void    tsdbDestroyCommitH(SCommitH *pCommith);
 static int32_t tsdbCreateCommitIters(SCommitH *pCommith);
 static void    tsdbDestroyCommitIters(SCommitH *pCommith);
 static int     tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid);
-static void    tsdbResetCommitFile(SCommitH *pCommith);
 static int     tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid);
 static int     tsdbCommitToTable(SCommitH *pCommith, int tid);
 static bool    tsdbCommitIsSameFile(SCommitH *pCommith, int bidx);
@@ -88,8 +93,11 @@ static void tsdbLoadAndMergeFromCache(STsdb *pTsdb, SDataCols *pDataCols, int *i
                                       SDataCols *pTarget, TSKEY maxKey, int maxRows, int8_t update);
 static int  tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf);
 static int  tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn);
+static int  tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, STbDataIter *pIter, TSKEY maxKey, int maxRowsToRead,
+                                  SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup,
+                                  SMergeInfo *pMergeInfo);
 
-int tsdbBegin(STsdb *pTsdb) {
+int32_t tsdbBegin(STsdb *pTsdb) {
   if (!pTsdb) return 0;
 
   SMemTable *pMem;
@@ -112,15 +120,50 @@ int32_t tsdbCommit(STsdb *pTsdb) {
   pTsdb->mem = NULL;
 
   // start commit
-  tsdbStartCommit(pTsdb);
-  if (tsdbInitCommitH(&commith, pTsdb) < 0) {
-    return -1;
+  code = tsdbStartCommit(pTsdb, &commith);
+  if (code) {
+    goto _err;
+  }
+
+  // commit impl
+  code = tsdbCommitData(&commith);
+  if (code) {
+    goto _err;
   }
 
+  code = tsdbCommitDel(&commith);
+  if (code) {
+    goto _err;
+  }
+
+  code = tsdbCommitCache(&commith);
+  if (code) {
+    goto _err;
+  }
+
+  // end commit
+  code = tsdbEndCommit(&commith, 0);
+  if (code) {
+    goto _err;
+  }
+
+  return code;
+
+_err:
+  tsdbError("vgId:%d failed to commit since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+static int32_t tsdbCommitData(SCommitH *pCommith) {
+  int32_t    fid;
+  SDFileSet *pSet = NULL;
+  int32_t    code = 0;
+  STsdb     *pTsdb = TSDB_COMMIT_REPO(pCommith);
+
   // Skip expired memory data and expired FSET
-  tsdbSeekCommitIter(&commith, commith.rtn.minKey);
-  while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) {
-    if (pSet->fid < commith.rtn.minFid) {
+  tsdbSeekCommitIter(pCommith, pCommith->rtn.minKey);
+  while ((pSet = tsdbFSIterNext(&(pCommith->fsIter)))) {
+    if (pSet->fid < pCommith->rtn.minFid) {
       tsdbInfo("vgId:%d, FSET %d on level %d disk id %d expires, remove it", REPO_ID(pTsdb), pSet->fid,
                TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet));
     } else {
@@ -129,7 +172,7 @@ int32_t tsdbCommit(STsdb *pTsdb) {
   }
 
   // commit
-  fid = tsdbNextCommitFid(&(commith));
+  fid = tsdbNextCommitFid(pCommith);
   while (true) {
     // Loop over both on disk and memory
     if (pSet == NULL && fid == TSDB_IVLD_FID) break;
@@ -137,12 +180,12 @@ int32_t tsdbCommit(STsdb *pTsdb) {
     if (pSet && (fid == TSDB_IVLD_FID || pSet->fid < fid)) {
       // Only has existing FSET but no memory data to commit in this
       // existing FSET, only check if file in correct retention
-      if (tsdbApplyRtnOnFSet(pTsdb, pSet, &(commith.rtn)) < 0) {
-        tsdbDestroyCommitH(&commith);
+      if (tsdbApplyRtnOnFSet(TSDB_COMMIT_REPO(pCommith), pSet, &(pCommith->rtn)) < 0) {
+        tsdbDestroyCommitH(pCommith);
         return -1;
       }
 
-      pSet = tsdbFSIterNext(&(commith.fsIter));
+      pSet = tsdbFSIterNext(&(pCommith->fsIter));
     } else {
       // Has memory data to commit
       SDFileSet *pCSet;
@@ -156,22 +199,30 @@ int32_t tsdbCommit(STsdb *pTsdb) {
         // Commit to an existing FSET
         pCSet = pSet;
         cfid = pSet->fid;
-        pSet = tsdbFSIterNext(&(commith.fsIter));
+        pSet = tsdbFSIterNext(&(pCommith->fsIter));
       }
 
-      if (tsdbCommitToFile(&commith, pCSet, cfid) < 0) {
-        tsdbDestroyCommitH(&commith);
+      if (tsdbCommitToFile(pCommith, pCSet, cfid) < 0) {
+        tsdbDestroyCommitH(pCommith);
         return -1;
       }
 
-      fid = tsdbNextCommitFid(&commith);
+      fid = tsdbNextCommitFid(pCommith);
     }
   }
 
-  // end commit
-  tsdbDestroyCommitH(&commith);
-  tsdbEndCommit(pTsdb, TSDB_CODE_SUCCESS);
+  return code;
+}
 
+static int32_t tsdbCommitDel(SCommitH *pCommith) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+static int32_t tsdbCommitCache(SCommitH *pCommith) {
+  int32_t code = 0;
+  // TODO
   return code;
 }
 
@@ -216,16 +267,6 @@ static int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) {
   return 0;
 }
 
-// int tsdbPrepareCommit(STsdb *pTsdb) {
-//   if (pTsdb->mem == NULL) return 0;
-
-//   ASSERT(pTsdb->imem == NULL);
-
-//   pTsdb->imem = pTsdb->mem;
-//   pTsdb->mem = NULL;
-//   return 0;
-// }
-
 void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) {
   STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pRepo);
   TSKEY         minKey, midKey, maxKey, now;
@@ -243,19 +284,32 @@ void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) {
             pRtn->minFid, pRtn->midFid, pRtn->maxFid);
 }
 
-static void tsdbStartCommit(STsdb *pRepo) {
-  SMemTable *pMem = pRepo->imem;
+static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitH *pCHandle) {
+  int32_t code = 0;
+
+  tsdbInfo("vgId:%d, start to commit", REPO_ID(pTsdb));
+
+  if (tsdbInitCommitH(pCHandle, pTsdb) < 0) {
+    return -1;
+  }
 
-  tsdbInfo("vgId:%d, start to commit", REPO_ID(pRepo));
+  tsdbStartFSTxn(pTsdb, 0, 0);
 
-  tsdbStartFSTxn(pRepo, 0, 0);
+  return code;
 }
 
-static void tsdbEndCommit(STsdb *pTsdb, int eno) {
+static int32_t tsdbEndCommit(SCommitH *pCHandle, int eno) {
+  int32_t code = 0;
+  STsdb  *pTsdb = TSDB_COMMIT_REPO(pCHandle);
+
+  tsdbDestroyCommitH(pCHandle);
   tsdbEndFSTxn(pTsdb);
   tsdbMemTableDestroy(pTsdb->imem);
   pTsdb->imem = NULL;
+
   tsdbInfo("vgId:%d, commit over, %s", REPO_ID(pTsdb), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed");
+
+  return code;
 }
 
 static int tsdbInitCommitH(SCommitH *pCommith, STsdb *pRepo) {
@@ -354,34 +408,73 @@ static void tsdbDestroyCommitH(SCommitH *pCommith) {
   tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith));
 }
 
-static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) {
-  STsdb        *pRepo = TSDB_COMMIT_REPO(pCommith);
+static int32_t tsdbCommitToFileStart(SCommitH *pCHandle, SDFileSet *pSet, int32_t fid) {
+  int32_t       code = 0;
+  STsdb        *pRepo = TSDB_COMMIT_REPO(pCHandle);
   STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pRepo);
 
   ASSERT(pSet == NULL || pSet->fid == fid);
 
-  tsdbResetCommitFile(pCommith);
-  tsdbGetFidKeyRange(pCfg->days, pCfg->precision, fid, &(pCommith->minKey), &(pCommith->maxKey));
+  pCHandle->fid = fid;
+  pCHandle->pSet = pSet;
+  pCHandle->isRFileSet = false;
+  pCHandle->isDFileSame = false;
+  pCHandle->isLFileSame = false;
+  taosArrayClear(pCHandle->aBlkIdx);
+
+  tsdbGetFidKeyRange(pCfg->days, pCfg->precision, fid, &(pCHandle->minKey), &(pCHandle->maxKey));
+
+  code = tsdbSetAndOpenCommitFile(pCHandle, pSet, fid);
 
-  // Set and open files
-  if (tsdbSetAndOpenCommitFile(pCommith, pSet, fid) < 0) {
+  return code;
+}
+static int32_t tsdbCommitToFileImpl(SCommitH *pCHandle) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+static int32_t tsdbCommitToFileEnd(SCommitH *pCommith) {
+  int32_t code = 0;
+  STsdb  *pRepo = TSDB_COMMIT_REPO(pCommith);
+
+  if (tsdbWriteBlockIdx(TSDB_COMMIT_HEAD_FILE(pCommith), pCommith->aBlkIdx, (void **)(&(TSDB_COMMIT_BUF(pCommith)))) <
+      0) {
+    tsdbError("vgId:%d, failed to write SBlockIdx part to FSET %d since %s", REPO_ID(pRepo), pCommith->fid,
+              tstrerror(terrno));
+    tsdbCloseCommitFile(pCommith, true);
+    // revert the file change
+    tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pCommith->pSet);
     return -1;
   }
-#if 0
-  // Loop to commit each table data
-  for (int tid = 0; tid < pCommith->niters; tid++) {
-    SCommitIter *pIter = pCommith->iters + tid;
 
-    if (pIter->pTable == NULL) continue;
+  if (tsdbUpdateDFileSetHeader(&(pCommith->wSet)) < 0) {
+    tsdbError("vgId:%d, failed to update FSET %d header since %s", REPO_ID(pRepo), pCommith->fid, tstrerror(terrno));
+    tsdbCloseCommitFile(pCommith, true);
+    // revert the file change
+    tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pCommith->pSet);
+    return -1;
+  }
 
-    if (tsdbCommitToTable(pCommith, tid) < 0) {
-      tsdbCloseCommitFile(pCommith, true);
-      // revert the file change
-      tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet);
-      return -1;
-    }
+  // Close commit file
+  tsdbCloseCommitFile(pCommith, false);
+
+  if (tsdbUpdateDFileSet(REPO_FS(pRepo), &(pCommith->wSet)) < 0) {
+    return -1;
   }
-#endif
+
+  return code;
+}
+static int32_t tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) {
+  int32_t       code = 0;
+  STsdb        *pRepo = TSDB_COMMIT_REPO(pCommith);
+  STsdbKeepCfg *pCfg = REPO_KEEP_CFG(pRepo);
+
+  // commit to file start
+  code = tsdbCommitToFileStart(pCommith, pSet, fid);
+  if (code) {
+    goto _err;
+  }
+
   // Loop to commit each table data in mem and file
   int mIter = 0, fIter = 0;
   int nBlkIdx = taosArrayGetSize(pCommith->readh.aBlkIdx);
@@ -426,31 +519,16 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) {
     }
   }
 
-  if (tsdbWriteBlockIdx(TSDB_COMMIT_HEAD_FILE(pCommith), pCommith->aBlkIdx, (void **)(&(TSDB_COMMIT_BUF(pCommith)))) <
-      0) {
-    tsdbError("vgId:%d, failed to write SBlockIdx part to FSET %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
-    tsdbCloseCommitFile(pCommith, true);
-    // revert the file change
-    tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet);
-    return -1;
-  }
-
-  if (tsdbUpdateDFileSetHeader(&(pCommith->wSet)) < 0) {
-    tsdbError("vgId:%d, failed to update FSET %d header since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
-    tsdbCloseCommitFile(pCommith, true);
-    // revert the file change
-    tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet);
-    return -1;
+  // commit to file end
+  code = tsdbCommitToFileEnd(pCommith);
+  if (code) {
+    goto _err;
   }
 
-  // Close commit file
-  tsdbCloseCommitFile(pCommith, false);
-
-  if (tsdbUpdateDFileSet(REPO_FS(pRepo), &(pCommith->wSet)) < 0) {
-    return -1;
-  }
+  return code;
 
-  return 0;
+_err:
+  return code;
 }
 
 static int32_t tsdbCreateCommitIters(SCommitH *pCommith) {
@@ -507,13 +585,6 @@ static void tsdbDestroyCommitIters(SCommitH *pCommith) {
   pCommith->niters = 0;
 }
 
-static void tsdbResetCommitFile(SCommitH *pCommith) {
-  pCommith->isRFileSet = false;
-  pCommith->isDFileSame = false;
-  pCommith->isLFileSame = false;
-  taosArrayClear(pCommith->aBlkIdx);
-}
-
 static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid) {
   SDiskID    did;
   STsdb     *pRepo = TSDB_COMMIT_REPO(pCommith);
@@ -1590,4 +1661,171 @@ static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *p
   }
 
   return false;
+}
+
+static int tsdbAppendTableRowToCols(STsdb *pTsdb, STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row,
+                                    bool merge) {
+  if (pCols) {
+    if (*ppSchema == NULL || schemaVersion(*ppSchema) != TD_ROW_SVER(row)) {
+      *ppSchema = tsdbGetTableSchemaImpl(pTsdb, pTable, false, false, TD_ROW_SVER(row));
+      if (*ppSchema == NULL) {
+        ASSERT(false);
+        return -1;
+      }
+    }
+
+    tdAppendSTSRowToDataCol(row, *ppSchema, pCols, merge);
+  }
+
+  return 0;
+}
+
+static int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, STbDataIter *pIter, TSKEY maxKey, int maxRowsToRead,
+                                 SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup,
+                                 SMergeInfo *pMergeInfo) {
+  ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0);
+  if (pIter == NULL) return 0;
+  STSchema *pSchema = NULL;
+  TSKEY     rowKey = 0;
+  TSKEY     fKey = 0;
+  // only fetch lastKey from mem data as file data not used in this function actually
+  TSKEY      lastKey = TSKEY_INITIAL_VAL;
+  bool       isRowDel = false;
+  int        filterIter = 0;
+  STSRow    *row = NULL;
+  SMergeInfo mInfo;
+
+  // TODO: support Multi-Version(the rows with the same TS keys in memory can't be merged if its version refered by
+  // query handle)
+
+  if (pMergeInfo == NULL) pMergeInfo = &mInfo;
+
+  memset(pMergeInfo, 0, sizeof(*pMergeInfo));
+  pMergeInfo->keyFirst = INT64_MAX;
+  pMergeInfo->keyLast = INT64_MIN;
+  if (pCols) tdResetDataCols(pCols);
+
+  row = tsdbNextIterRow(pIter);
+  if (row == NULL || TD_ROW_KEY(row) > maxKey) {
+    rowKey = INT64_MAX;
+    isRowDel = false;
+  } else {
+    rowKey = TD_ROW_KEY(row);
+    isRowDel = TD_ROW_IS_DELETED(row);
+  }
+
+  if (filterIter >= nFilterKeys) {
+    fKey = INT64_MAX;
+  } else {
+    fKey = tdGetKey(filterKeys[filterIter]);
+  }
+  // 1. fkey - no dup since merged up to maxVersion of each query handle by tsdbLoadBlockDataCols
+  // 2. rowKey - would dup since Multi-Version supported
+  while (true) {
+    if (fKey == INT64_MAX && rowKey == INT64_MAX) break;
+
+    if (fKey < rowKey) {
+      pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey);
+      pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey);
+
+      filterIter++;
+      if (filterIter >= nFilterKeys) {
+        fKey = INT64_MAX;
+      } else {
+        fKey = tdGetKey(filterKeys[filterIter]);
+      }
+#if 1
+    } else if (fKey > rowKey) {
+      if (isRowDel) {
+        // TODO: support delete function
+        pMergeInfo->rowsDeleteFailed++;
+      } else {
+        if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break;
+        if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+
+        if (lastKey != rowKey) {
+          pMergeInfo->rowsInserted++;
+          pMergeInfo->nOperations++;
+          pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
+          pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
+          if (pCols) {
+            if (lastKey != TSKEY_INITIAL_VAL) {
+              ++pCols->numOfRows;
+            }
+            tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, false);
+          }
+          lastKey = rowKey;
+        } else {
+          if (keepDup) {
+            tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, true);
+          } else {
+            // discard
+          }
+        }
+      }
+
+      tsdbTbDataIterNext(pIter);
+      row = tsdbNextIterRow(pIter);
+      if (row == NULL || TD_ROW_KEY(row) > maxKey) {
+        rowKey = INT64_MAX;
+        isRowDel = false;
+      } else {
+        rowKey = TD_ROW_KEY(row);
+        isRowDel = TD_ROW_IS_DELETED(row);
+      }
+    } else {           // fkey == rowKey
+      if (isRowDel) {  // TODO: support delete function(How to stands for delete in file? rowVersion = -1?)
+        ASSERT(!keepDup);
+        if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+        pMergeInfo->rowsDeleteSucceed++;
+        pMergeInfo->nOperations++;
+        tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, false);
+      } else {
+        if (keepDup) {
+          if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
+          if (lastKey != rowKey) {
+            pMergeInfo->rowsUpdated++;
+            pMergeInfo->nOperations++;
+            pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
+            pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
+            if (pCols) {
+              if (lastKey != TSKEY_INITIAL_VAL) {
+                ++pCols->numOfRows;
+              }
+              tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, false);
+            }
+            lastKey = rowKey;
+          } else {
+            tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, true);
+          }
+        } else {
+          pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey);
+          pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey);
+        }
+      }
+
+      tsdbTbDataIterNext(pIter);
+      row = tsdbNextIterRow(pIter);
+      if (row == NULL || TD_ROW_KEY(row) > maxKey) {
+        rowKey = INT64_MAX;
+        isRowDel = false;
+      } else {
+        rowKey = TD_ROW_KEY(row);
+        isRowDel = TD_ROW_IS_DELETED(row);
+      }
+
+      filterIter++;
+      if (filterIter >= nFilterKeys) {
+        fKey = INT64_MAX;
+      } else {
+        fKey = tdGetKey(filterKeys[filterIter]);
+      }
+    }
+#endif
+  }
+  if (pCols && (lastKey != TSKEY_INITIAL_VAL)) {
+    ++pCols->numOfRows;
+  }
+
+  return 0;
 }
\ No newline at end of file

source/dnode/vnode/src/tsdb/tsdbCommit2.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "tsdb.h"
-
-typedef struct {
-  SMemTable2 *pMemTable;
-  int32_t     minutes;
-  int8_t      precision;
-  TSKEY       nCommitKey;
-  int32_t     fid;
-  TSKEY       minKey;
-  TSKEY       maxKey;
-  SReadH      readh;
-  SDFileSet   wSet;
-  SArray     *aBlkIdx;
-  SArray     *aSupBlk;
-  SArray     *aSubBlk;
-  SArray     *aDelInfo;
-} SCommitH;
-
-static int32_t tsdbCommitStart(SCommitH *pCHandle, STsdb *pTsdb);
-static int32_t tsdbCommitEnd(SCommitH *pCHandle);
-static int32_t tsdbCommitImpl(SCommitH *pCHandle);
-
-int32_t tsdbBegin2(STsdb *pTsdb) {
-  int32_t code = 0;
-
-  ASSERT(pTsdb->mem == NULL);
-  code = tsdbMemTableCreate2(pTsdb, (SMemTable2 **)&pTsdb->mem);
-  if (code) {
-    tsdbError("vgId:%d failed to begin TSDB since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
-    goto _exit;
-  }
-
-_exit:
-  return code;
-}
-
-int32_t tsdbCommit2(STsdb *pTsdb) {
-  int32_t  code = 0;
-  SCommitH ch = {0};
-
-  // start to commit
-  code = tsdbCommitStart(&ch, pTsdb);
-  if (code) {
-    goto _exit;
-  }
-
-  // commit
-  code = tsdbCommitImpl(&ch);
-  if (code) {
-    goto _err;
-  }
-
-  // end commit
-  code = tsdbCommitEnd(&ch);
-  if (code) {
-    goto _exit;
-  }
-
-_exit:
-  return code;
-
-_err:
-  tsdbError("vgId:%d failed to commit since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
-  return code;
-}
-
-static int32_t tsdbCommitStart(SCommitH *pCHandle, STsdb *pTsdb) {
-  int32_t     code = 0;
-  SMemTable2 *pMemTable = (SMemTable2 *)pTsdb->mem;
-
-  tsdbInfo("vgId:%d start to commit", TD_VID(pTsdb->pVnode));
-
-  // switch to commit
-  ASSERT(pTsdb->imem == NULL && pTsdb->mem);
-  pTsdb->imem = pTsdb->mem;
-  pTsdb->mem = NULL;
-
-  // open handle
-  pCHandle->pMemTable = pMemTable;
-  pCHandle->minutes = pTsdb->keepCfg.days;
-  pCHandle->precision = pTsdb->keepCfg.precision;
-  pCHandle->nCommitKey = pMemTable->minKey.ts;
-
-  code = tsdbInitReadH(&pCHandle->readh, pTsdb);
-  if (code) {
-    goto _err;
-  }
-  pCHandle->aBlkIdx = taosArrayInit(0, sizeof(SBlockIdx));
-  if (pCHandle->aBlkIdx == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _err;
-  }
-  pCHandle->aSupBlk = taosArrayInit(0, sizeof(SBlock));
-  if (pCHandle->aSupBlk == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _err;
-  }
-  pCHandle->aSubBlk = taosArrayInit(0, sizeof(SBlock));
-  if (pCHandle->aSubBlk == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _err;
-  }
-  pCHandle->aDelInfo = taosArrayInit(0, sizeof(SDelInfo));
-  if (pCHandle->aDelInfo == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _err;
-  }
-
-  // start FS transaction
-  tsdbStartFSTxn(pTsdb, 0, 0);
-
-  return code;
-
-_err:
-  return code;
-}
-
-static int32_t tsdbCommitEnd(SCommitH *pCHandle) {
-  int32_t     code = 0;
-  STsdb      *pTsdb = pCHandle->pMemTable->pTsdb;
-  SMemTable2 *pMemTable = (SMemTable2 *)pTsdb->imem;
-
-  // end transaction
-  code = tsdbEndFSTxn(pTsdb);
-  if (code) {
-    goto _err;
-  }
-
-  // close handle
-  taosArrayClear(pCHandle->aDelInfo);
-  taosArrayClear(pCHandle->aSubBlk);
-  taosArrayClear(pCHandle->aSupBlk);
-  taosArrayClear(pCHandle->aBlkIdx);
-  tsdbDestroyReadH(&pCHandle->readh);
-
-  // destroy memtable (todo: unref it)
-  pTsdb->imem = NULL;
-  tsdbMemTableDestroy2(pMemTable);
-
-  tsdbInfo("vgId:%d commit over", TD_VID(pTsdb->pVnode));
-  return code;
-
-_err:
-  return code;
-}
-
-static int32_t tsdbCommitTableStart(SCommitH *pCHandle) {
-  int32_t code = 0;
-  // TODO
-  return code;
-}
-
-static int32_t tsdbCommitTableEnd(SCommitH *pCHandle) {
-  int32_t code = 0;
-  // TODO
-  return code;
-}
-
-static int32_t tsdbCommitTable(SCommitH *pCHandle, SMemData *pMemData, SBlockIdx *pBlockIdx) {
-  int32_t      code = 0;
-  SMemDataIter iter = {0};
-
-  // commit table start
-  code = tsdbCommitTableStart(pCHandle);
-  if (code) {
-    goto _err;
-  }
-
-  // commit table impl
-  if (pMemData && pBlockIdx) {
-    // TODO
-  } else if (pMemData) {
-    // TODO
-  } else {
-    // TODO
-  }
-
-  // commit table end
-  code = tsdbCommitTableEnd(pCHandle);
-  if (code) {
-    goto _err;
-  }
-
-  return code;
-
-_err:
-  return code;
-}
-
-static int32_t tsdbTableIdCmprFn(const void *p1, const void *p2) {
-  TABLEID *pId1 = (TABLEID *)p1;
-  TABLEID *pId2 = (TABLEID *)p2;
-
-  if (pId1->suid < pId2->suid) {
-    return -1;
-  } else if (pId1->suid > pId2->suid) {
-    return 1;
-  }
-
-  if (pId1->uid < pId2->uid) {
-    return -1;
-  } else if (pId1->uid > pId2->uid) {
-    return 1;
-  }
-
-  return 0;
-}
-
-static int32_t tsdbWriteBlockIdx(SDFile *pFile, SArray *pArray, uint8_t **ppBuf) {
-  int32_t code = 0;
-  // TODO
-  return code;
-}
-
-static int32_t tsdbCommitFileStart(SCommitH *pCHandle) {
-  int32_t    code = 0;
-  STsdb     *pTsdb = pCHandle->pMemTable->pTsdb;
-  SDFileSet *pSet = NULL;
-
-  taosArrayClear(pCHandle->aBlkIdx);
-
-  return code;
-}
-
-static int32_t tsdbCommitFileEnd(SCommitH *pCHandle) {
-  int32_t code = 0;
-  // TODO
-  return code;
-}
-
-static int32_t tsdbCommitFile(SCommitH *pCHandle) {
-  int32_t    code = 0;
-  SMemData  *pMemData;
-  SBlockIdx *pBlockIdx;
-  int32_t    iMemData;
-  int32_t    nMemData;
-  int32_t    iBlockIdx;
-  int32_t    nBlockIdx;
-
-  // commit file start
-  code = tsdbCommitFileStart(pCHandle);
-  if (code) {
-    goto _err;
-  }
-
-  // commit file impl
-  iMemData = 0;
-  nMemData = taosArrayGetSize(pCHandle->pMemTable->aMemData);
-  iBlockIdx = 0;
-  nBlockIdx = 0;  // todo
-
-  for (;;) {
-    if (iMemData >= nMemData && iBlockIdx >= nBlockIdx) break;
-
-    pMemData = NULL;
-    pBlockIdx = NULL;
-    if (iMemData < nMemData) {
-      pMemData = (SMemData *)taosArrayGetP(pCHandle->pMemTable->aMemData, iMemData);
-    }
-    if (iBlockIdx < nBlockIdx) {
-      // pBlockIdx = ;
-    }
-
-    if (pMemData && pBlockIdx) {
-      int32_t c = tsdbTableIdCmprFn(pMemData, pBlockIdx);
-      if (c < 0) {
-        iMemData++;
-        pBlockIdx = NULL;
-      } else if (c == 0) {
-        iMemData++;
-        iBlockIdx++;
-      } else {
-        iBlockIdx++;
-        pMemData = NULL;
-      }
-    } else {
-      if (pMemData) {
-        iMemData++;
-      } else {
-        iBlockIdx++;
-      }
-    }
-
-    code = tsdbCommitTable(pCHandle, pMemData, pBlockIdx);
-    if (code) {
-      goto _err;
-    }
-  }
-
-  // commit file end
-  code = tsdbCommitFileEnd(pCHandle);
-  if (code) {
-    goto _err;
-  }
-
-  return code;
-
-_err:
-  return code;
-}
-
-static int32_t tsdbCommitData(SCommitH *pCHandle) {
-  int32_t code = 0;
-  int32_t fid;
-
-  if (pCHandle->pMemTable->nRows == 0) goto _exit;
-
-  // loop to commit to each file
-  for (;;) {
-    if (pCHandle->nCommitKey == TSKEY_MAX) break;
-
-    pCHandle->fid = TSDB_KEY_FID(pCHandle->nCommitKey, pCHandle->minutes, pCHandle->precision);
-    tsdbGetFidKeyRange(pCHandle->minutes, pCHandle->precision, pCHandle->fid, &pCHandle->minKey, &pCHandle->maxKey);
-    code = tsdbCommitFile(pCHandle);
-    if (code) {
-      goto _err;
-    }
-  }
-
-_exit:
-  return code;
-
-_err:
-  return code;
-}
-
-static int32_t delInfoCmprFn(const void *p1, const void *p2) {
-  SDelInfo *pDelInfo1 = (SDelInfo *)p1;
-  SDelInfo *pDelInfo2 = (SDelInfo *)p2;
-
-  if (pDelInfo1->suid < pDelInfo2->suid) {
-    return -1;
-  } else if (pDelInfo1->suid > pDelInfo2->suid) {
-    return 1;
-  }
-
-  if (pDelInfo1->uid < pDelInfo2->uid) {
-    return -1;
-  } else if (pDelInfo1->uid > pDelInfo2->uid) {
-    return 1;
-  }
-
-  if (pDelInfo1->version < pDelInfo2->version) {
-    return -1;
-  } else if (pDelInfo1->version > pDelInfo2->version) {
-    return 1;
-  }
-
-  return 0;
-}
-static int32_t tsdbCommitDelete(SCommitH *pCHandle) {
-  int32_t   code = 0;
-  SDelInfo  delInfo;
-  SMemData *pMemData;
-
-  if (pCHandle->pMemTable->nDelOp == 0) goto _exit;
-
-  // load del array (todo)
-
-  // loop to append SDelInfo
-  for (int32_t iMemData = 0; iMemData < taosArrayGetSize(pCHandle->pMemTable->aMemData); iMemData++) {
-    pMemData = (SMemData *)taosArrayGetP(pCHandle->pMemTable->aMemData, iMemData);
-
-    for (SDelOp *pDelOp = pMemData->delOpHead; pDelOp; pDelOp = pDelOp->pNext) {
-      delInfo = (SDelInfo){.suid = pMemData->suid,
-                           .uid = pMemData->uid,
-                           .version = pDelOp->version,
-                           .sKey = pDelOp->sKey,
-                           .eKey = pDelOp->eKey};
-      if (taosArrayPush(pCHandle->aDelInfo, &delInfo) == NULL) {
-        code = TSDB_CODE_OUT_OF_MEMORY;
-        goto _err;
-      }
-    }
-  }
-
-  taosArraySort(pCHandle->aDelInfo, delInfoCmprFn);
-
-  // write to new file
-
-_exit:
-  return code;
-
-_err:
-  return code;
-}
-
-static int32_t tsdbCommitCache(SCommitH *pCHandle) {
-  int32_t code = 0;
-  // TODO
-  return code;
-}
-
-static int32_t tsdbCommitImpl(SCommitH *pCHandle) {
-  int32_t code = 0;
-
-  // commit data
-  code = tsdbCommitData(pCHandle);
-  if (code) {
-    goto _err;
-  }
-
-  // commit delete
-  code = tsdbCommitDelete(pCHandle);
-  if (code) {
-    goto _err;
-  }
-
-  // commit cache if need (todo)
-  if (0) {
-    code = tsdbCommitCache(pCHandle);
-    if (code) {
-      goto _err;
-    }
-  }
-
-  return code;
-
-_err:
-  return code;
-}
\ No newline at end of file

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -188,23 +188,6 @@ _err:
   return code;
 }
 
-static int tsdbAppendTableRowToCols(STsdb *pTsdb, STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row,
-                                    bool merge) {
-  if (pCols) {
-    if (*ppSchema == NULL || schemaVersion(*ppSchema) != TD_ROW_SVER(row)) {
-      *ppSchema = tsdbGetTableSchemaImpl(pTsdb, pTable, false, false, TD_ROW_SVER(row));
-      if (*ppSchema == NULL) {
-        ASSERT(false);
-        return -1;
-      }
-    }
-
-    tdAppendSTSRowToDataCol(row, *ppSchema, pCols, merge);
-  }
-
-  return 0;
-}
-
 int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter) {
   int32_t code = 0;
 
@@ -310,166 +293,6 @@ bool tsdbTbDataIterGet(STbDataIter *pIter, TSDBROW *pRow) {
   return true;
 }
 
-/**
- * This is an important function to load data or try to load data from memory skiplist iterator.
- *
- * This function load memory data until:
- * 1. iterator ends
- * 2. data key exceeds maxKey
- * 3. rowsIncreased = rowsInserted - rowsDeleteSucceed >= maxRowsToRead
- * 4. operations in pCols not exceeds its max capacity if pCols is given
- *
- * The function tries to procceed AS MUCH AS POSSIBLE.
- */
-int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, STbDataIter *pIter, TSKEY maxKey, int maxRowsToRead,
-                          SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) {
-  ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0);
-  if (pIter == NULL) return 0;
-  STSchema *pSchema = NULL;
-  TSKEY     rowKey = 0;
-  TSKEY     fKey = 0;
-  // only fetch lastKey from mem data as file data not used in this function actually
-  TSKEY      lastKey = TSKEY_INITIAL_VAL;
-  bool       isRowDel = false;
-  int        filterIter = 0;
-  STSRow    *row = NULL;
-  SMergeInfo mInfo;
-
-  // TODO: support Multi-Version(the rows with the same TS keys in memory can't be merged if its version refered by
-  // query handle)
-
-  if (pMergeInfo == NULL) pMergeInfo = &mInfo;
-
-  memset(pMergeInfo, 0, sizeof(*pMergeInfo));
-  pMergeInfo->keyFirst = INT64_MAX;
-  pMergeInfo->keyLast = INT64_MIN;
-  if (pCols) tdResetDataCols(pCols);
-
-  row = tsdbNextIterRow(pIter);
-  if (row == NULL || TD_ROW_KEY(row) > maxKey) {
-    rowKey = INT64_MAX;
-    isRowDel = false;
-  } else {
-    rowKey = TD_ROW_KEY(row);
-    isRowDel = TD_ROW_IS_DELETED(row);
-  }
-
-  if (filterIter >= nFilterKeys) {
-    fKey = INT64_MAX;
-  } else {
-    fKey = tdGetKey(filterKeys[filterIter]);
-  }
-  // 1. fkey - no dup since merged up to maxVersion of each query handle by tsdbLoadBlockDataCols
-  // 2. rowKey - would dup since Multi-Version supported
-  while (true) {
-    if (fKey == INT64_MAX && rowKey == INT64_MAX) break;
-
-    if (fKey < rowKey) {
-      pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey);
-      pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey);
-
-      filterIter++;
-      if (filterIter >= nFilterKeys) {
-        fKey = INT64_MAX;
-      } else {
-        fKey = tdGetKey(filterKeys[filterIter]);
-      }
-#if 1
-    } else if (fKey > rowKey) {
-      if (isRowDel) {
-        // TODO: support delete function
-        pMergeInfo->rowsDeleteFailed++;
-      } else {
-        if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break;
-        if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
-
-        if (lastKey != rowKey) {
-          pMergeInfo->rowsInserted++;
-          pMergeInfo->nOperations++;
-          pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
-          pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
-          if (pCols) {
-            if (lastKey != TSKEY_INITIAL_VAL) {
-              ++pCols->numOfRows;
-            }
-            tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, false);
-          }
-          lastKey = rowKey;
-        } else {
-          if (keepDup) {
-            tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, true);
-          } else {
-            // discard
-          }
-        }
-      }
-
-      tsdbTbDataIterNext(pIter);
-      row = tsdbNextIterRow(pIter);
-      if (row == NULL || TD_ROW_KEY(row) > maxKey) {
-        rowKey = INT64_MAX;
-        isRowDel = false;
-      } else {
-        rowKey = TD_ROW_KEY(row);
-        isRowDel = TD_ROW_IS_DELETED(row);
-      }
-    } else {           // fkey == rowKey
-      if (isRowDel) {  // TODO: support delete function(How to stands for delete in file? rowVersion = -1?)
-        ASSERT(!keepDup);
-        if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
-        pMergeInfo->rowsDeleteSucceed++;
-        pMergeInfo->nOperations++;
-        tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, false);
-      } else {
-        if (keepDup) {
-          if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
-          if (lastKey != rowKey) {
-            pMergeInfo->rowsUpdated++;
-            pMergeInfo->nOperations++;
-            pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
-            pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
-            if (pCols) {
-              if (lastKey != TSKEY_INITIAL_VAL) {
-                ++pCols->numOfRows;
-              }
-              tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, false);
-            }
-            lastKey = rowKey;
-          } else {
-            tsdbAppendTableRowToCols(pTsdb, pTable, pCols, &pSchema, row, true);
-          }
-        } else {
-          pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey);
-          pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey);
-        }
-      }
-
-      tsdbTbDataIterNext(pIter);
-      row = tsdbNextIterRow(pIter);
-      if (row == NULL || TD_ROW_KEY(row) > maxKey) {
-        rowKey = INT64_MAX;
-        isRowDel = false;
-      } else {
-        rowKey = TD_ROW_KEY(row);
-        isRowDel = TD_ROW_IS_DELETED(row);
-      }
-
-      filterIter++;
-      if (filterIter >= nFilterKeys) {
-        fKey = INT64_MAX;
-      } else {
-        fKey = tdGetKey(filterKeys[filterIter]);
-      }
-    }
-#endif
-  }
-  if (pCols && (lastKey != TSKEY_INITIAL_VAL)) {
-    ++pCols->numOfRows;
-  }
-
-  return 0;
-}
-
 static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData) {
   int32_t  code = 0;
   int32_t  idx = 0;

source/dnode/vnode/src/tsdb/tsdbMemTable2.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "tsdb.h"
-
-typedef struct {
-  tb_uid_t  uid;
-  STSchema *pTSchema;
-} SSkmInfo;
-
-#define SL_MAX_LEVEL 5
-
-#define SL_NODE_SIZE(l)        (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2)
-#define SL_NODE_FORWARD(n, l)  ((n)->forwards[l])
-#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)])
-#define SL_NODE_DATA(n)        (&SL_NODE_BACKWARD(n, (n)->level))
-
-#define SL_MOVE_BACKWARD 0x1
-#define SL_MOVE_FROM_POS 0x2
-
-static int32_t tsdbGetOrCreateMemData(SMemTable2 *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData);
-static int     memDataPCmprFn(const void *p1, const void *p2);
-static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow);
-static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow);
-static int8_t  tsdbMemSkipListRandLevel(SMemSkipList *pSl);
-static int32_t tsdbInsertTableDataImpl(SMemTable2 *pMemTable, SMemData *pMemData, int64_t version,
-                                       SVSubmitBlk *pSubmitBlk);
-static void    memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags);
-
-// SMemTable ==============================================
-int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable2 **ppMemTable) {
-  int32_t     code = 0;
-  SMemTable2 *pMemTable = NULL;
-
-  pMemTable = (SMemTable2 *)taosMemoryCalloc(1, sizeof(*pMemTable));
-  if (pMemTable == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _err;
-  }
-  pMemTable->pTsdb = pTsdb;
-  pMemTable->nRef = 1;
-  pMemTable->minKey = (TSDBKEY){.version = INT64_MAX, .ts = TSKEY_MAX};
-  pMemTable->maxKey = (TSDBKEY){.version = -1, .ts = TSKEY_MIN};
-  pMemTable->nRows = 0;
-  pMemTable->nDelOp = 0;
-  pMemTable->aMemData = taosArrayInit(512, sizeof(SMemData *));
-  if (pMemTable->aMemData == NULL) {
-    taosMemoryFree(pMemTable);
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _err;
-  }
-
-  *ppMemTable = pMemTable;
-  return code;
-
-_err:
-  *ppMemTable = NULL;
-  return code;
-}
-
-void tsdbMemTableDestroy2(SMemTable2 *pMemTable) {
-  taosArrayDestroyEx(pMemTable->aMemData, NULL /*TODO*/);
-  taosMemoryFree(pMemTable);
-}
-
-int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmitBlk) {
-  int32_t     code = 0;
-  SMemTable2 *pMemTable = (SMemTable2 *)pTsdb->mem;  // TODO
-  SMemData   *pMemData;
-  TSDBROW     row = {.version = version};
-
-  ASSERT(pMemTable);
-  ASSERT(pSubmitBlk->nData > 0);
-
-  {
-    // check if table exists (todo)
-  }
-
-  code = tsdbGetOrCreateMemData(pMemTable, pSubmitBlk->suid, pSubmitBlk->uid, &pMemData);
-  if (code) {
-    tsdbError("vgId:%d, failed to create/get table data since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
-    goto _err;
-  }
-
-  // do insert
-  code = tsdbInsertTableDataImpl(pMemTable, pMemData, version, pSubmitBlk);
-  if (code) {
-    goto _err;
-  }
-
-  return code;
-
-_err:
-  return code;
-}
-
-int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey) {
-  int32_t     code = 0;
-  SMemTable2 *pMemTable = (SMemTable2 *)pTsdb->mem;  // TODO
-  SMemData   *pMemData;
-  SVBufPool  *pPool = pTsdb->pVnode->inUse;
-
-  ASSERT(pMemTable);
-
-  {
-    // check if table exists (todo)
-  }
-
-  code = tsdbGetOrCreateMemData(pMemTable, suid, uid, &pMemData);
-  if (code) {
-    goto _err;
-  }
-
-  // do delete
-  SDelOp *pDelOp = (SDelOp *)vnodeBufPoolMalloc(pPool, sizeof(*pDelOp));
-  if (pDelOp == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _err;
-  }
-  pDelOp->version = version;
-  pDelOp->sKey = sKey;
-  pDelOp->eKey = eKey;
-  pDelOp->pNext = NULL;
-  if (pMemData->delOpHead == NULL) {
-    ASSERT(pMemData->delOpTail == NULL);
-    pMemData->delOpHead = pMemData->delOpTail = pDelOp;
-  } else {
-    pMemData->delOpTail->pNext = pDelOp;
-    pMemData->delOpTail = pDelOp;
-  }
-
-  {
-    // update the state of pMemTable, pMemData, last and lastrow (todo)
-  }
-
-  pMemTable->nDelOp++;
-
-  tsdbDebug("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " sKey:%" PRId64 " eKey:%" PRId64
-            " since %s",
-            TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code));
-  return code;
-
-_err:
-  tsdbError("vgId:%d, failed to delete data from table suid:%" PRId64 " uid:%" PRId64 " sKey:%" PRId64 " eKey:%" PRId64
-            " since %s",
-            TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code));
-  return code;
-}
-
-void tsdbMemDataIterOpen(SMemData *pMemData, TSDBKEY *pKey, int8_t backward, SMemDataIter *pIter) {
-  SMemSkipListNode *pos[SL_MAX_LEVEL];
-
-  pIter->pMemData = pMemData;
-  pIter->backward = backward;
-  pIter->pRow = NULL;
-  if (pKey == NULL) {
-    // create from head or tail
-    if (backward) {
-      pIter->pNode = SL_NODE_BACKWARD(pMemData->sl.pTail, 0);
-    } else {
-      pIter->pNode = SL_NODE_FORWARD(pMemData->sl.pHead, 0);
-    }
-  } else {
-    // create from a key
-    if (backward) {
-      memDataMovePosTo(pMemData, pos, pKey, SL_MOVE_BACKWARD);
-      pIter->pNode = SL_NODE_BACKWARD(pos[0], 0);
-    } else {
-      memDataMovePosTo(pMemData, pos, pKey, 0);
-      pIter->pNode = SL_NODE_FORWARD(pos[0], 0);
-    }
-  }
-}
-
-bool tsdbMemDataIterNext(SMemDataIter *pIter) {
-  SMemSkipListNode *pHead = pIter->pMemData->sl.pHead;
-  SMemSkipListNode *pTail = pIter->pMemData->sl.pTail;
-
-  pIter->pRow = NULL;
-  if (pIter->backward) {
-    ASSERT(pIter->pNode != pTail);
-
-    if (pIter->pNode == pHead) {
-      return false;
-    }
-
-    pIter->pNode = SL_NODE_BACKWARD(pIter->pNode, 0);
-    if (pIter->pNode == pHead) {
-      return false;
-    }
-  } else {
-    ASSERT(pIter->pNode != pHead);
-
-    if (pIter->pNode == pTail) {
-      return false;
-    }
-
-    pIter->pNode = SL_NODE_FORWARD(pIter->pNode, 0);
-    if (pIter->pNode == pTail) {
-      return false;
-    }
-  }
-
-  return true;
-}
-
-void tsdbMemDataIterGet(SMemDataIter *pIter, TSDBROW **ppRow) {
-  if (pIter->pRow) {
-    *ppRow = pIter->pRow;
-  } else {
-    SMemSkipListNode *pHead = pIter->pMemData->sl.pHead;
-    SMemSkipListNode *pTail = pIter->pMemData->sl.pTail;
-
-    if (pIter->backward) {
-      ASSERT(pIter->pNode != pTail);
-
-      if (pIter->pNode == pHead) {
-        *ppRow = NULL;
-      } else {
-        tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), &pIter->row);
-        *ppRow = &pIter->row;
-      }
-    } else {
-      ASSERT(pIter->pNode != pHead);
-
-      if (pIter->pNode == pTail) {
-        *ppRow = NULL;
-      } else {
-        tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), &pIter->row);
-        *ppRow = &pIter->row;
-      }
-    }
-  }
-}
-
-static int32_t tsdbGetOrCreateMemData(SMemTable2 *pMemTable, tb_uid_t suid, tb_uid_t uid, SMemData **ppMemData) {
-  int32_t    code = 0;
-  int32_t    idx = 0;
-  SMemData  *pMemDataT = &(SMemData){.suid = suid, .uid = uid};
-  SMemData  *pMemData = NULL;
-  SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse;
-  int8_t     maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel;
-
-  // get
-  idx = taosArraySearchIdx(pMemTable->aMemData, &pMemDataT, memDataPCmprFn, TD_GE);
-  if (idx >= 0) {
-    pMemData = (SMemData *)taosArrayGet(pMemTable->aMemData, idx);
-    if (memDataPCmprFn(&pMemDataT, &pMemData) == 0) goto _exit;
-  }
-
-  // create
-  pMemData = vnodeBufPoolMalloc(pPool, sizeof(*pMemData) + SL_NODE_SIZE(maxLevel) * 2);
-  if (pMemData == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _err;
-  }
-  pMemData->suid = suid;
-  pMemData->uid = uid;
-  pMemData->minKey = (TSDBKEY){.version = INT64_MAX, .ts = TSKEY_MAX};
-  pMemData->maxKey = (TSDBKEY){.version = -1, .ts = TSKEY_MIN};
-  pMemData->delOpHead = pMemData->delOpTail = NULL;
-  pMemData->sl.seed = taosRand();
-  pMemData->sl.size = 0;
-  pMemData->sl.maxLevel = maxLevel;
-  pMemData->sl.level = 0;
-  pMemData->sl.pHead = (SMemSkipListNode *)&pMemData[1];
-  pMemData->sl.pTail = (SMemSkipListNode *)POINTER_SHIFT(pMemData->sl.pHead, SL_NODE_SIZE(maxLevel));
-  pMemData->sl.pHead->level = maxLevel;
-  pMemData->sl.pTail->level = maxLevel;
-
-  for (int8_t iLevel = 0; iLevel < pMemData->sl.maxLevel; iLevel++) {
-    SL_NODE_FORWARD(pMemData->sl.pHead, iLevel) = pMemData->sl.pTail;
-    SL_NODE_BACKWARD(pMemData->sl.pHead, iLevel) = NULL;
-    SL_NODE_BACKWARD(pMemData->sl.pTail, iLevel) = pMemData->sl.pHead;
-    SL_NODE_FORWARD(pMemData->sl.pTail, iLevel) = NULL;
-  }
-
-  if (idx < 0) idx = 0;
-  if (taosArrayInsert(pMemTable->aMemData, idx, &pMemData) == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _err;
-  }
-
-_exit:
-  *ppMemData = pMemData;
-  return code;
-
-_err:
-  *ppMemData = NULL;
-  return code;
-}
-
-static int memDataPCmprFn(const void *p1, const void *p2) {
-  SMemData *pMemData1 = *(SMemData **)p1;
-  SMemData *pMemData2 = *(SMemData **)p2;
-
-  if (pMemData1->suid < pMemData2->suid) {
-    return -1;
-  } else if (pMemData1->suid > pMemData2->suid) {
-    return 1;
-  }
-
-  if (pMemData1->uid < pMemData2->uid) {
-    return -1;
-  } else if (pMemData1->uid > pMemData2->uid) {
-    return 1;
-  }
-
-  return 0;
-}
-
-static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow) {
-  int32_t n = 0;
-
-  n += tPutI64(p ? p + n : p, pRow->version);
-  n += tPutTSRow(p ? p + n : p, &pRow->tsRow);
-
-  return n;
-}
-
-static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow) {
-  int32_t n = 0;
-
-  n += tGetI64(p + n, &pRow->version);
-  n += tGetTSRow(p + n, &pRow->tsRow);
-
-  return n;
-}
-
-static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) {
-  int8_t         level = 1;
-  int8_t         tlevel = TMIN(pSl->maxLevel, pSl->level + 1);
-  const uint32_t factor = 4;
-
-  while ((taosRandR(&pSl->seed) % factor) == 0 && level < tlevel) {
-    level++;
-  }
-
-  return level;
-}
-
-static void memDataMovePosTo(SMemData *pMemData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags) {
-  SMemSkipListNode *px;
-  SMemSkipListNode *pn;
-  TSDBKEY          *pTKey;
-  int               c;
-  int               backward = flags & SL_MOVE_BACKWARD;
-  int               fromPos = flags & SL_MOVE_FROM_POS;
-
-  if (backward) {
-    px = pMemData->sl.pTail;
-
-    for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= pMemData->sl.level; iLevel--) {
-      pos[iLevel] = px;
-    }
-
-    if (pMemData->sl.level) {
-      if (fromPos) px = pos[pMemData->sl.level - 1];
-
-      for (int8_t iLevel = pMemData->sl.level - 1; iLevel >= 0; iLevel--) {
-        pn = SL_NODE_BACKWARD(px, iLevel);
-        while (pn != pMemData->sl.pHead) {
-          pTKey = (TSDBKEY *)SL_NODE_DATA(pn);
-
-          c = tsdbKeyCmprFn(pTKey, pKey);
-          if (c <= 0) {
-            break;
-          } else {
-            px = pn;
-            pn = SL_NODE_BACKWARD(px, iLevel);
-          }
-        }
-
-        pos[iLevel] = px;
-      }
-    }
-  } else {
-    px = pMemData->sl.pHead;
-
-    for (int8_t iLevel = pMemData->sl.maxLevel - 1; iLevel >= pMemData->sl.level; iLevel--) {
-      pos[iLevel] = px;
-    }
-
-    if (pMemData->sl.level) {
-      if (fromPos) px = pos[pMemData->sl.level - 1];
-
-      for (int8_t iLevel = pMemData->sl.level - 1; iLevel >= 0; iLevel--) {
-        pn = SL_NODE_FORWARD(px, iLevel);
-        while (pn != pMemData->sl.pHead) {
-          pTKey = (TSDBKEY *)SL_NODE_DATA(pn);
-
-          c = tsdbKeyCmprFn(pTKey, pKey);
-          if (c >= 0) {
-            break;
-          } else {
-            px = pn;
-            pn = SL_NODE_FORWARD(px, iLevel);
-          }
-        }
-
-        pos[iLevel] = px;
-      }
-    }
-  }
-}
-
-static int32_t memDataDoPut(SMemTable2 *pMemTable, SMemData *pMemData, SMemSkipListNode **pos, TSDBROW *pRow,
-                            int8_t forward) {
-  int32_t           code = 0;
-  int8_t            level;
-  SMemSkipListNode *pNode;
-  SVBufPool        *pPool = pMemTable->pTsdb->pVnode->inUse;
-
-  // node
-  level = tsdbMemSkipListRandLevel(&pMemData->sl);
-  pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level) + tPutTSDBRow(NULL, pRow));
-  if (pNode == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _exit;
-  }
-  pNode->level = level;
-  for (int8_t iLevel = 0; iLevel < level; iLevel++) {
-    SL_NODE_FORWARD(pNode, iLevel) = NULL;
-    SL_NODE_BACKWARD(pNode, iLevel) = NULL;
-  }
-
-  tPutTSDBRow((uint8_t *)SL_NODE_DATA(pNode), pRow);
-
-  // put
-  for (int8_t iLevel = 0; iLevel < pNode->level; iLevel++) {
-    SMemSkipListNode *px = pos[iLevel];
-
-    if (forward) {
-      SMemSkipListNode *pNext = SL_NODE_FORWARD(px, iLevel);
-
-      SL_NODE_FORWARD(pNode, iLevel) = pNext;
-      SL_NODE_BACKWARD(pNode, iLevel) = px;
-
-      SL_NODE_BACKWARD(pNext, iLevel) = pNode;
-      SL_NODE_FORWARD(px, iLevel) = pNode;
-    } else {
-      SMemSkipListNode *pPrev = SL_NODE_BACKWARD(px, iLevel);
-
-      SL_NODE_FORWARD(pNode, iLevel) = px;
-      SL_NODE_BACKWARD(pNode, iLevel) = pPrev;
-
-      SL_NODE_FORWARD(pPrev, iLevel) = pNode;
-      SL_NODE_BACKWARD(px, iLevel) = pNode;
-    }
-  }
-
-  pMemData->sl.size++;
-  if (pMemData->sl.level < pNode->level) {
-    pMemData->sl.level = pNode->level;
-  }
-
-_exit:
-  return code;
-}
-
-static int32_t tsdbInsertTableDataImpl(SMemTable2 *pMemTable, SMemData *pMemData, int64_t version,
-                                       SVSubmitBlk *pSubmitBlk) {
-  int32_t  code = 0;
-  int32_t  n = 0;
-  uint8_t *p = pSubmitBlk->pData;
-  int32_t  nRow = 0;
-  TSDBROW  row = {.version = version};
-
-  SMemSkipListNode *pos[SL_MAX_LEVEL];
-
-  ASSERT(pSubmitBlk->nData);
-
-  // backward put first data
-  n += tGetTSRow(p + n, &row.tsRow);
-  ASSERT(n <= pSubmitBlk->nData);
-
-  memDataMovePosTo(pMemData, pos, &(TSDBKEY){.version = version, .ts = row.tsRow.ts}, SL_MOVE_BACKWARD);
-  code = memDataDoPut(pMemTable, pMemData, pos, &row, 0);
-  if (code) {
-    goto _exit;
-  }
-  nRow++;
-
-  if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemData->minKey) < 0) {
-    pMemData->minKey = *(TSDBKEY *)&row;
-  }
-  if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemTable->minKey) < 0) {
-    pMemTable->minKey = *(TSDBKEY *)&row;
-  }
-
-  // forward put rest
-  for (int8_t iLevel = 0; iLevel < pMemData->sl.maxLevel; iLevel++) {
-    pos[iLevel] = SL_NODE_BACKWARD(pos[iLevel], iLevel);
-  }
-  while (n < pSubmitBlk->nData) {
-    n += tGetTSRow(p + n, &row.tsRow);
-    ASSERT(n <= pSubmitBlk->nData);
-
-    memDataMovePosTo(pMemData, pos, &(TSDBKEY){.version = version, .ts = row.tsRow.ts}, SL_MOVE_FROM_POS);
-    code = memDataDoPut(pMemTable, pMemData, pos, &row, 1);
-    if (code) {
-      goto _exit;
-    }
-
-    nRow++;
-  }
-
-  if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemData->maxKey) > 0) {
-    pMemData->maxKey = *(TSDBKEY *)&row;
-  }
-  if (tsdbKeyCmprFn((TSDBKEY *)&row, &pMemTable->maxKey) > 0) {
-    pMemTable->maxKey = *(TSDBKEY *)&row;
-  }
-  pMemTable->nRows += nRow;
-
-_exit:
-  return code;
-}

source/dnode/vnode/src/tsdb/tsdbReadImpl.c

@@ -196,33 +196,6 @@ int tsdbSetReadTable(SReadH *pReadh, STable *pTable) {
   } else {
     pReadh->pBlkIdx = (SBlockIdx *)p;
   }
-  // size_t size = taosArrayGetSize(pReadh->aBlkIdx);
-  // if (size > 0) {
-  //   while (true) {
-  //     if (pReadh->cidx >= size) {
-  //       pReadh->pBlkIdx = NULL;
-  //       break;
-  //     }
-
-  //     SBlockIdx *pBlkIdx = taosArrayGet(pReadh->aBlkIdx, pReadh->cidx);
-  //     if (pBlkIdx->uid == TABLE_TID(pTable)) {
-  //       if (pBlkIdx->uid == TABLE_UID(pTable)) {
-  //         pReadh->pBlkIdx = pBlkIdx;
-  //       } else {
-  //         pReadh->pBlkIdx = NULL;
-  //       }
-  //       pReadh->cidx++;
-  //       break;
-  //     } else if (pBlkIdx->uid > TABLE_TID(pTable)) {
-  //       pReadh->pBlkIdx = NULL;
-  //       break;
-  //     } else {
-  //       pReadh->cidx++;
-  //     }
-  //   }
-  // } else {
-  //   pReadh->pBlkIdx = NULL;
-  // }
 
   return 0;
 }

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -81,7 +81,9 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
   pVnode->state.applied = info.state.committed;
   pVnode->pTfs = pTfs;
   pVnode->msgCb = msgCb;
+  pVnode->syncCount = 0;
 
+  tsem_init(&pVnode->syncSem, 0, 0);
   tsem_init(&(pVnode->canCommit), 0, 1);
 
   // open buffer pool
@@ -175,6 +177,7 @@ void vnodeClose(SVnode *pVnode) {
     vnodeCloseBufPool(pVnode);
     // destroy handle
     tsem_destroy(&(pVnode->canCommit));
+    tsem_destroy(&pVnode->syncSem);
     taosMemoryFree(pVnode);
   }
 }

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -91,9 +91,6 @@ int32_t vnodePreprocessReq(SVnode *pVnode, SRpcMsg *pMsg) {
       }
 
     } break;
-    case TDMT_VND_ALTER_REPLICA: {
-      code = vnodeSyncAlter(pVnode, pMsg);
-    } break;
     default:
       break;
   }
@@ -107,7 +104,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
   int32_t len;
   int32_t ret;
 
-  vTrace("vgId:%d, start to process write request %s, version %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
+  vTrace("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType),
          version);
 
   pVnode->state.applied = version;
@@ -173,7 +170,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
       break;
   }
 
-  vTrace("vgId:%d, process %s request success, version: %" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version);
+  vTrace("vgId:%d, process %s request success, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version);
 
   if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) {
     vError("vgId:%d, failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno));

source/dnode/vnode/src/vnd/vnodeSync.c

@@ -16,41 +16,41 @@
 #define _DEFAULT_SOURCE
 #include "vnd.h"
 
-static int32_t   vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg);
-static int32_t   vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg);
-static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode);
-static void      vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
-static void      vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
-static void      vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
-static int32_t   vnodeSyncGetSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot);
+static inline bool vnodeIsMsgBlock(tmsg_t type) {
+  return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_CONFIRM) || (type == TDMT_VND_ALTER_REPLICA);
+}
 
-int32_t vnodeSyncOpen(SVnode *pVnode, char *path) {
-  SSyncInfo syncInfo = {
-      .vgId = pVnode->config.vgId,
-      .isStandBy = pVnode->config.standby,
-      .syncCfg = pVnode->config.syncCfg,
-      .pWal = pVnode->pWal,
-      .msgcb = NULL,
-      .FpSendMsg = vnodeSyncSendMsg,
-      .FpEqMsg = vnodeSyncEqMsg,
-  };
+static inline bool vnodeIsMsgWeak(tmsg_t type) { return false; }
 
-  snprintf(syncInfo.path, sizeof(syncInfo.path), "%s%ssync", path, TD_DIRSEP);
-  syncInfo.pFsm = vnodeSyncMakeFsm(pVnode);
+static inline void vnodeAccumBlockMsg(SVnode *pVnode, tmsg_t type) {
+  if (!vnodeIsMsgBlock(type)) return;
 
-  pVnode->sync = syncOpen(&syncInfo);
-  if (pVnode->sync <= 0) {
-    vError("vgId:%d, failed to open sync since %s", pVnode->config.vgId, terrstr());
-    return -1;
-  }
+  int32_t count = atomic_add_fetch_32(&pVnode->syncCount, 1);
+  vTrace("vgId:%d, accum block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type));
+}
 
-  setPingTimerMS(pVnode->sync, 3000);
-  setElectTimerMS(pVnode->sync, 500);
-  setHeartbeatTimerMS(pVnode->sync, 100);
-  return 0;
+static inline void vnodeWaitBlockMsg(SVnode *pVnode) {
+  int32_t count = atomic_load_32(&pVnode->syncCount);
+  if (count <= 0) return;
+
+  vTrace("vgId:%d, wait block finish, count:%d", pVnode->config.vgId, count);
+  tsem_wait(&pVnode->syncSem);
+}
+
+static inline void vnodePostBlockMsg(SVnode *pVnode, tmsg_t type) {
+  if (!vnodeIsMsgBlock(type)) return;
+
+  int32_t count = atomic_load_32(&pVnode->syncCount);
+  if (count <= 0) return;
+
+  count = atomic_sub_fetch_32(&pVnode->syncCount, 1);
+  vTrace("vgId:%d, post block, count:%d type:%s", pVnode->config.vgId, count, TMSG_INFO(type));
+  if (count <= 0) {
+    tsem_post(&pVnode->syncSem);
+  }
 }
 
-int32_t vnodeSyncAlter(SVnode *pVnode, SRpcMsg *pMsg) {
+static int32_t vnodeProcessSyncReconfigReq(SVnode *pVnode, SRpcMsg *pMsg) {
   SAlterVnodeReq req = {0};
   if (tDeserializeSAlterVnodeReq((char *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead), &req) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
@@ -66,29 +66,98 @@ int32_t vnodeSyncAlter(SVnode *pVnode, SRpcMsg *pMsg) {
     vInfo("vgId:%d, replica:%d %s:%u", TD_VID(pVnode), r, pNode->nodeFqdn, pNode->nodePort);
   }
 
-  int32_t code = syncReconfig(pVnode->sync, &cfg);
-  if (code == TAOS_SYNC_PROPOSE_SUCCESS) {
-    // todo refactor
-    SRpcMsg rsp = {.info = pMsg->info, .code = 0};
-    tmsgSendRsp(&rsp);
-    return TSDB_CODE_ACTION_IN_PROGRESS;
+  return syncReconfig(pVnode->sync, &cfg);
+}
+
+void vnodeProposeMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
+  SVnode  *pVnode = pInfo->ahandle;
+  int32_t  vgId = pVnode->config.vgId;
+  int32_t  code = 0;
+  SRpcMsg *pMsg = NULL;
+
+  for (int32_t m = 0; m < numOfMsgs; m++) {
+    if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
+    vTrace("vgId:%d, msg:%p get from vnode-write queue handle:%p", vgId, pMsg, pMsg->info.handle);
+
+    if (pMsg->msgType == TDMT_VND_ALTER_REPLICA) {
+      code = vnodeProcessSyncReconfigReq(pVnode, pMsg);
+    } else {
+      code = vnodePreprocessReq(pVnode, pMsg);
+      if (code != 0) {
+        vError("vgId:%d, failed to pre-process msg:%p since %s", vgId, pMsg, terrstr());
+      } else {
+        code = syncPropose(pVnode->sync, pMsg, vnodeIsMsgWeak(pMsg->msgType));
+      }
+    }
+
+    if (code == 0) {
+      vnodeAccumBlockMsg(pVnode, pMsg->msgType);
+      if (pMsg->msgType == TDMT_VND_ALTER_REPLICA) {
+        // todo refactor
+        SRpcMsg rsp = {.code = code, .info = pMsg->info};
+        tmsgSendRsp(&rsp);
+      }
+    } else if (code == TAOS_SYNC_PROPOSE_NOT_LEADER) {
+      SEpSet newEpSet = {0};
+      syncGetEpSet(pVnode->sync, &newEpSet);
+      SEp *pEp = &newEpSet.eps[newEpSet.inUse];
+      if (pEp->port == tsServerPort && strcmp(pEp->fqdn, tsLocalFqdn) == 0) {
+        newEpSet.inUse = (newEpSet.inUse + 1) % newEpSet.numOfEps;
+      }
+
+      vTrace("vgId:%d, msg:%p is redirect since not leader, numOfEps:%d inUse:%d", vgId, pMsg, newEpSet.numOfEps,
+             newEpSet.inUse);
+      for (int32_t i = 0; i < newEpSet.numOfEps; ++i) {
+        vTrace("vgId:%d, msg:%p redirect:%d ep:%s:%u", vgId, pMsg, i, newEpSet.eps[i].fqdn, newEpSet.eps[i].port);
+      }
+
+      SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info};
+      tmsgSendRedirectRsp(&rsp, &newEpSet);
+    } else {
+      if (terrno != 0) code = terrno;
+      vError("vgId:%d, msg:%p failed to propose since %s, code:0x%x", vgId, pMsg, tstrerror(code), code);
+      SRpcMsg rsp = {.code = code, .info = pMsg->info};
+      tmsgSendRsp(&rsp);
+    }
+
+    vTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, code);
+    rpcFreeCont(pMsg->pCont);
+    taosFreeQitem(pMsg);
   }
 
-  return code;
+  vnodeWaitBlockMsg(pVnode);
 }
 
-void vnodeSyncStart(SVnode *pVnode) {
-  syncSetMsgCb(pVnode->sync, &pVnode->msgCb);
-  if (pVnode->config.standby) {
-    syncStartStandBy(pVnode->sync);
-  } else {
-    syncStart(pVnode->sync);
+void vnodeApplyMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
+  SVnode  *pVnode = pInfo->ahandle;
+  int32_t  vgId = pVnode->config.vgId;
+  int32_t  code = 0;
+  SRpcMsg *pMsg = NULL;
+
+  for (int32_t i = 0; i < numOfMsgs; ++i) {
+    if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
+    vTrace("vgId:%d, msg:%p get from vnode-apply queue, handle:%p", vgId, pMsg, pMsg->info.handle);
+
+    SRpcMsg rsp = {.code = pMsg->code, .info = pMsg->info};
+    if (rsp.code == 0) {
+      if (vnodeProcessWriteReq(pVnode, pMsg, pMsg->conn.applyIndex, &rsp) < 0) {
+        rsp.code = terrno;
+        vError("vgId:%d, msg:%p failed to apply since %s", vgId, pMsg, terrstr());
+      }
+    }
+
+    vnodePostBlockMsg(pVnode, pMsg->msgType);
+    if (rsp.info.handle != NULL) {
+      tmsgSendRsp(&rsp);
+    }
+
+    vTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, rsp.code);
+    rpcFreeCont(pMsg->pCont);
+    taosFreeQitem(pMsg);
   }
 }
 
-void vnodeSyncClose(SVnode *pVnode) { syncStop(pVnode->sync); }
-
-int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
+static int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
   int32_t code = tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg);
   if (code != 0) {
     rpcFreeCont(pMsg->pCont);
@@ -97,7 +166,7 @@ int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
   return code;
 }
 
-int32_t vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
+static int32_t vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
   int32_t code = tmsgSendReq(pEpSet, pMsg);
   if (code != 0) {
     rpcFreeCont(pMsg->pCont);
@@ -106,53 +175,43 @@ int32_t vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
   return code;
 }
 
-int32_t vnodeSyncGetSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) {
+static int32_t vnodeSyncGetSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) {
   vnodeGetSnapshot(pFsm->data, pSnapshot);
   return 0;
 }
 
-void vnodeSyncReconfig(struct SSyncFSM *pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) {
+static void vnodeSyncReconfig(struct SSyncFSM *pFsm, SSyncCfg newCfg, SReConfigCbMeta cbMeta) {
   SVnode *pVnode = pFsm->data;
   vInfo("vgId:%d, sync reconfig is confirmed", TD_VID(pVnode));
 
   // todo rpc response here
+  // build rpc msg
+  // put into apply queue
 }
 
-void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
+static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
+  SVnode   *pVnode = pFsm->data;
+  SSnapshot snapshot = {0};
   SyncIndex beginIndex = SYNC_INDEX_INVALID;
+  char      logBuf[256] = {0};
+
   if (pFsm->FpGetSnapshot != NULL) {
-    SSnapshot snapshot = {0};
-    pFsm->FpGetSnapshot(pFsm, &snapshot);
+    (*pFsm->FpGetSnapshot)(pFsm, &snapshot);
     beginIndex = snapshot.lastApplyIndex;
   }
 
   if (cbMeta.index > beginIndex) {
-    char logBuf[256] = {0};
     snprintf(
         logBuf, sizeof(logBuf),
         "==callback== ==CommitCb== execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, beginIndex :%ld\n",
         pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex);
     syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
 
-    SVnode       *pVnode = pFsm->data;
-    SyncApplyMsg *pSyncApplyMsg = syncApplyMsgBuild2(pMsg, pVnode->config.vgId, &cbMeta);
-    SRpcMsg       applyMsg;
-    syncApplyMsg2RpcMsg(pSyncApplyMsg, &applyMsg);
-    syncApplyMsgDestroy(pSyncApplyMsg);
-
-    // recover handle for response
-    SRpcMsg saveRpcMsg;
-    int32_t ret = syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &saveRpcMsg);
-    if (ret == 1 && cbMeta.state == TAOS_SYNC_STATE_LEADER) {
-      applyMsg.info = saveRpcMsg.info;
-    } else {
-      applyMsg.info.handle = NULL;
-      applyMsg.info.ahandle = NULL;
-    }
-
-    // put to applyQ
-    tmsgPutToQueue(&(pVnode->msgCb), APPLY_QUEUE, &applyMsg);
-
+    SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen, .conn.applyIndex = cbMeta.index};
+    rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
+    memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen);
+    syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info);
+    tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, &rpcMsg);
   } else {
     char logBuf[256] = {0};
     snprintf(logBuf, sizeof(logBuf),
@@ -164,7 +223,7 @@ void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta)
   }
 }
 
-void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
+static void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
   char logBuf[256] = {0};
   snprintf(logBuf, sizeof(logBuf),
            "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index,
@@ -172,14 +231,14 @@ void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMet
   syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
 }
 
-void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
+static void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
   char logBuf[256] = {0};
   snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n",
            pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
   syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
 }
 
-SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
+static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
   SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM));
   pFsm->data = pVnode;
   pFsm->FpCommitCb = vnodeSyncCommitMsg;
@@ -188,6 +247,42 @@ SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
   pFsm->FpGetSnapshot = vnodeSyncGetSnapshot;
   pFsm->FpRestoreFinishCb = NULL;
   pFsm->FpReConfigCb = vnodeSyncReconfig;
-
   return pFsm;
-}
+}
\ No newline at end of file
+
+int32_t vnodeSyncOpen(SVnode *pVnode, char *path) {
+  SSyncInfo syncInfo = {
+      .vgId = pVnode->config.vgId,
+      .isStandBy = pVnode->config.standby,
+      .syncCfg = pVnode->config.syncCfg,
+      .pWal = pVnode->pWal,
+      .msgcb = NULL,
+      .FpSendMsg = vnodeSyncSendMsg,
+      .FpEqMsg = vnodeSyncEqMsg,
+  };
+
+  snprintf(syncInfo.path, sizeof(syncInfo.path), "%s%ssync", path, TD_DIRSEP);
+  syncInfo.pFsm = vnodeSyncMakeFsm(pVnode);
+
+  pVnode->sync = syncOpen(&syncInfo);
+  if (pVnode->sync <= 0) {
+    vError("vgId:%d, failed to open sync since %s", pVnode->config.vgId, terrstr());
+    return -1;
+  }
+
+  setPingTimerMS(pVnode->sync, 3000);
+  setElectTimerMS(pVnode->sync, 500);
+  setHeartbeatTimerMS(pVnode->sync, 100);
+  return 0;
+}
+
+void vnodeSyncStart(SVnode *pVnode) {
+  syncSetMsgCb(pVnode->sync, &pVnode->msgCb);
+  if (pVnode->config.standby) {
+    syncStartStandBy(pVnode->sync);
+  } else {
+    syncStart(pVnode->sync);
+  }
+}
+
+void vnodeSyncClose(SVnode *pVnode) { syncStop(pVnode->sync); }

source/libs/catalog/inc/catalogInt.h

@@ -173,7 +173,6 @@ typedef struct SCtgJob {
   SArray*          pTasks;
   int32_t          taskDone;
   SMetaData        jobRes;
-  int32_t          rspCode;
 
   uint64_t         queryId;
   SCatalog*        pCtg; 
@@ -201,11 +200,12 @@ typedef struct SCtgMsgCtx {
 
 typedef struct SCtgTask {
   CTG_TASK_TYPE type;
-  int32_t  taskId;
-  SCtgJob *pJob;
-  void* taskCtx;
-  SCtgMsgCtx msgCtx;
-  void* res;
+  int32_t       taskId;
+  SCtgJob*      pJob;
+  void*         taskCtx;
+  SCtgMsgCtx    msgCtx;
+  int32_t       code;
+  void*         res;
 } SCtgTask;
 
 typedef int32_t (*ctgLanchTaskFp)(SCtgTask*);

source/libs/catalog/src/ctgAsync.c

@@ -437,13 +437,14 @@ _return:
 int32_t ctgDumpTbMetaRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pTableMeta) {
-    pJob->jobRes.pTableMeta = taosArrayInit(pJob->tbMetaNum, POINTER_BYTES);
+    pJob->jobRes.pTableMeta = taosArrayInit(pJob->tbMetaNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pTableMeta) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pTableMeta, &pTask->res);
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pTableMeta, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -451,14 +452,14 @@ int32_t ctgDumpTbMetaRes(SCtgTask* pTask) {
 int32_t ctgDumpDbVgRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pDbVgroup) {
-    pJob->jobRes.pDbVgroup = taosArrayInit(pJob->dbVgNum, POINTER_BYTES);
+    pJob->jobRes.pDbVgroup = taosArrayInit(pJob->dbVgNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pDbVgroup) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pDbVgroup, &pTask->res);
-  pTask->res = NULL;
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pDbVgroup, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -466,13 +467,14 @@ int32_t ctgDumpDbVgRes(SCtgTask* pTask) {
 int32_t ctgDumpTbHashRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pTableHash) {
-    pJob->jobRes.pTableHash = taosArrayInit(pJob->tbHashNum, sizeof(SVgroupInfo));
+    pJob->jobRes.pTableHash = taosArrayInit(pJob->tbHashNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pTableHash) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pTableHash, pTask->res);
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pTableHash, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -480,21 +482,29 @@ int32_t ctgDumpTbHashRes(SCtgTask* pTask) {
 int32_t ctgDumpIndexRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pIndex) {
-    pJob->jobRes.pIndex = taosArrayInit(pJob->indexNum, sizeof(SIndexInfo));
+    pJob->jobRes.pIndex = taosArrayInit(pJob->indexNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pIndex) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pIndex, pTask->res);
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pIndex, &res);
 
   return TSDB_CODE_SUCCESS;
 }
 
 int32_t ctgDumpQnodeRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
+  if (NULL == pJob->jobRes.pQnodeList) {
+    pJob->jobRes.pQnodeList = taosArrayInit(1, sizeof(SMetaRes));
+    if (NULL == pJob->jobRes.pQnodeList) {
+      CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
+    }
+  }
 
-  TSWAP(pJob->jobRes.pQnodeList, pTask->res); 
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pQnodeList, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -502,13 +512,14 @@ int32_t ctgDumpQnodeRes(SCtgTask* pTask) {
 int32_t ctgDumpDbCfgRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pDbCfg) {
-    pJob->jobRes.pDbCfg = taosArrayInit(pJob->dbCfgNum, sizeof(SDbCfgInfo));
+    pJob->jobRes.pDbCfg = taosArrayInit(pJob->dbCfgNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pDbCfg) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pDbCfg, pTask->res);
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pDbCfg, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -516,13 +527,14 @@ int32_t ctgDumpDbCfgRes(SCtgTask* pTask) {
 int32_t ctgDumpDbInfoRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pDbInfo) {
-    pJob->jobRes.pDbInfo = taosArrayInit(pJob->dbInfoNum, sizeof(SDbInfo));
+    pJob->jobRes.pDbInfo = taosArrayInit(pJob->dbInfoNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pDbInfo) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pDbInfo, pTask->res);
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pDbInfo, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -530,13 +542,14 @@ int32_t ctgDumpDbInfoRes(SCtgTask* pTask) {
 int32_t ctgDumpUdfRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pUdfList) {
-    pJob->jobRes.pUdfList = taosArrayInit(pJob->udfNum, sizeof(SFuncInfo));
+    pJob->jobRes.pUdfList = taosArrayInit(pJob->udfNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pUdfList) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pUdfList, pTask->res);
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pUdfList, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -544,13 +557,14 @@ int32_t ctgDumpUdfRes(SCtgTask* pTask) {
 int32_t ctgDumpUserRes(SCtgTask* pTask) {
   SCtgJob* pJob = pTask->pJob;
   if (NULL == pJob->jobRes.pUser) {
-    pJob->jobRes.pUser = taosArrayInit(pJob->userNum, sizeof(bool));
+    pJob->jobRes.pUser = taosArrayInit(pJob->userNum, sizeof(SMetaRes));
     if (NULL == pJob->jobRes.pUser) {
       CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
     }
   }
 
-  taosArrayPush(pJob->jobRes.pUser, pTask->res);
+  SMetaRes res = {.code = pTask->code, .pRes = pTask->res};
+  taosArrayPush(pJob->jobRes.pUser, &res);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -559,20 +573,13 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) {
   SCtgJob* pJob = pTask->pJob;
   int32_t code = 0;
 
-  qDebug("QID:%" PRIx64 " task %d end with rsp %s", pJob->queryId, pTask->taskId, tstrerror(rspCode));
+  qDebug("QID:0x%" PRIx64 " task %d end with rsp %s", pJob->queryId, pTask->taskId, tstrerror(rspCode));
 
-  if (rspCode) {
-    int32_t lastCode = atomic_val_compare_exchange_32(&pJob->rspCode, 0, rspCode);
-    if (0 == lastCode) {
-      CTG_ERR_JRET(rspCode);
-    }
-
-    return TSDB_CODE_SUCCESS;  
-  }
+  pTask->code = rspCode;
 
   int32_t taskDone = atomic_add_fetch_32(&pJob->taskDone, 1);
   if (taskDone < taosArrayGetSize(pJob->pTasks)) {
-    qDebug("task done: %d, total: %d", taskDone, (int32_t)taosArrayGetSize(pJob->pTasks));
+    qDebug("QID:0x%" PRIx64 " task done: %d, total: %d", pJob->queryId, taskDone, (int32_t)taosArrayGetSize(pJob->pTasks));
     return TSDB_CODE_SUCCESS;
   }
 

source/libs/catalog/src/ctgUtil.c

@@ -47,11 +47,13 @@ char *ctgTaskTypeStr(CTG_TASK_TYPE type) {
 void ctgFreeSMetaData(SMetaData* pData) {
   taosArrayDestroy(pData->pTableMeta);
   pData->pTableMeta = NULL;
-  
+
+/*  
   for (int32_t i = 0; i < taosArrayGetSize(pData->pDbVgroup); ++i) {
     SArray** pArray = taosArrayGet(pData->pDbVgroup, i);
     taosArrayDestroy(*pArray);
   }
+*/
   taosArrayDestroy(pData->pDbVgroup);
   pData->pDbVgroup = NULL;
   
@@ -61,10 +63,12 @@ void ctgFreeSMetaData(SMetaData* pData) {
   taosArrayDestroy(pData->pUdfList);
   pData->pUdfList = NULL;
 
+/*
   for (int32_t i = 0; i < taosArrayGetSize(pData->pDbCfg); ++i) {
     SDbCfgInfo* pInfo = taosArrayGet(pData->pDbCfg, i);
     taosArrayDestroy(pInfo->pRetensions);
   }
+*/  
   taosArrayDestroy(pData->pDbCfg);
   pData->pDbCfg = NULL;
 
@@ -320,8 +324,12 @@ void ctgFreeTask(SCtgTask* pTask) {
       break;
     }
     case CTG_TASK_GET_DB_CFG: {
-      taosMemoryFreeClear(pTask->taskCtx);      
-      taosMemoryFreeClear(pTask->res);
+      taosMemoryFreeClear(pTask->taskCtx);
+      if (pTask->res) {
+        SDbCfgInfo* pInfo = (SDbCfgInfo*)pTask->res;
+        taosArrayDestroy(pInfo->pRetensions);
+        taosMemoryFreeClear(pTask->res);
+      }
       break;
     }
     case CTG_TASK_GET_DB_INFO: {

source/libs/command/inc/commandInt.h

@@ -115,9 +115,9 @@ typedef struct SExplainCtx {
 #define EXPLAIN_ROW_NEW(level, ...)                                                                               \
   do {                                                                                                            \
     if (isVerboseLine) {                                                                                          \
-      tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, "%*s", (level) * 2 + 3, "");       \
+      tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, "%*s", (level) * 3 + 3, "");       \
     } else {                                                                                                      \
-      tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, "%*s%s", (level) * 2, "", "-> ");  \
+      tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, "%*s%s", (level) * 3, "", "-> ");  \
     }                                                                                                             \
     tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE - tlen, __VA_ARGS__);         \
   } while (0)

source/libs/command/src/command.c

@@ -28,91 +28,94 @@ static int32_t getSchemaBytes(const SSchema* pSchema) {
   }
 }
 
-// todo : to convert data according to SSDatablock
-static void buildRspData(const STableMeta* pMeta, char* pData) {
-  int32_t* payloadLen = (int32_t*) pData;
-  uint64_t* groupId = (uint64_t*)(pData + sizeof(int32_t));
+static SSDataBlock* buildDescResultDataBlock() {
+  SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
+  pBlock->info.numOfCols = DESCRIBE_RESULT_COLS;
+  pBlock->info.hasVarCol = true;
 
-  int32_t* pColSizes = (int32_t*)(pData + sizeof(int32_t) + sizeof(uint64_t));
-  pData = (char*) pColSizes + DESCRIBE_RESULT_COLS * sizeof(int32_t);
+  pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData));
 
-  int32_t numOfRows = TABLE_TOTAL_COL_NUM(pMeta);
+  SColumnInfoData infoData = {0};
+  infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
+  infoData.info.bytes = DESCRIBE_RESULT_FIELD_LEN;
 
-  // Field
-  int32_t* pOffset = (int32_t*)pData;
-  pData += numOfRows * sizeof(int32_t);
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+
+  infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
+  infoData.info.bytes = DESCRIBE_RESULT_TYPE_LEN;
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+
+  infoData.info.type = TSDB_DATA_TYPE_INT;
+  infoData.info.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes;;
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+
+  infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
+  infoData.info.bytes = DESCRIBE_RESULT_NOTE_LEN;
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+
+  return pBlock;
+}
+
+static void setDescResultIntoDataBlock(SSDataBlock* pBlock, int32_t numOfRows, STableMeta* pMeta) {
+  blockDataEnsureCapacity(pBlock, numOfRows);
+  pBlock->info.rows = numOfRows;
+
+  // field
+  SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0);
+  char buf[DESCRIBE_RESULT_FIELD_LEN] = {0};
   for (int32_t i = 0; i < numOfRows; ++i) {
-    STR_TO_VARSTR(pData, pMeta->schema[i].name);
-    int16_t len = varDataTLen(pData);
-    pData += len;
-    *pOffset = pColSizes[0];
-    pOffset += 1;
-    pColSizes[0] += len;
+    STR_TO_VARSTR(buf, pMeta->schema[i].name);
+    colDataAppend(pCol1, i, buf, false);
   }
-  
+
   // Type
-  pOffset = (int32_t*)pData;
-  pData += numOfRows * sizeof(int32_t);
+  SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1);
   for (int32_t i = 0; i < numOfRows; ++i) {
-    STR_TO_VARSTR(pData, tDataTypes[pMeta->schema[i].type].name);
-    int16_t len = varDataTLen(pData);
-    pData += len;
-    *pOffset = pColSizes[1];
-    pOffset += 1;
-    pColSizes[1] += len;
+    STR_TO_VARSTR(buf, tDataTypes[pMeta->schema[i].type].name);
+    colDataAppend(pCol2, i, buf, false);
   }
 
   // Length
-  pData += BitmapLen(numOfRows);
+  SColumnInfoData* pCol3 = taosArrayGet(pBlock->pDataBlock, 2);
   for (int32_t i = 0; i < numOfRows; ++i) {
-    *(int32_t*)pData = getSchemaBytes(pMeta->schema + i);
-    pData += sizeof(int32_t);
+    int32_t bytes = getSchemaBytes(pMeta->schema + i);
+    colDataAppend(pCol3, i, (const char*)&bytes, false);
   }
-  pColSizes[2] = sizeof(int32_t) * numOfRows;
 
   // Note
-  pOffset = (int32_t*)pData;
-  pData += numOfRows * sizeof(int32_t);
+  SColumnInfoData* pCol4 = taosArrayGet(pBlock->pDataBlock, 3);
   for (int32_t i = 0; i < numOfRows; ++i) {
-    STR_TO_VARSTR(pData, i >= pMeta->tableInfo.numOfColumns ? "TAG" : "");
-    int16_t len = varDataTLen(pData);
-    pData += len;
-    *pOffset = pColSizes[3];
-    pOffset += 1;
-    pColSizes[3] += len;
-  }
-
-  for (int32_t i = 0; i < DESCRIBE_RESULT_COLS; ++i) {
-    pColSizes[i] = htonl(pColSizes[i]);
+    STR_TO_VARSTR(buf, i >= pMeta->tableInfo.numOfColumns ? "TAG" : "");
+    colDataAppend(pCol4, i, buf, false);
   }
-
-
-  *payloadLen = (int32_t)(pData - (char*)payloadLen);
-}
-
-static int32_t calcRspSize(const STableMeta* pMeta) {
-  int32_t numOfRows = TABLE_TOTAL_COL_NUM(pMeta);
-  return sizeof(SRetrieveTableRsp) + 
-      (numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_FIELD_LEN) + 
-      (numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_TYPE_LEN) +
-      (BitmapLen(numOfRows) + numOfRows * sizeof(int32_t)) + 
-      (numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_NOTE_LEN) +
-      sizeof(int32_t) + sizeof(uint64_t);
 }
 
 static int32_t execDescribe(SNode* pStmt, SRetrieveTableRsp** pRsp) {
-  SDescribeStmt* pDesc = (SDescribeStmt*)pStmt;
-  *pRsp = taosMemoryCalloc(1, calcRspSize(pDesc->pMeta));
+  SDescribeStmt* pDesc = (SDescribeStmt*) pStmt;
+  int32_t numOfRows = TABLE_TOTAL_COL_NUM(pDesc->pMeta);
+
+  SSDataBlock* pBlock = buildDescResultDataBlock();
+  setDescResultIntoDataBlock(pBlock, numOfRows, pDesc->pMeta);
+
+  size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
+  *pRsp = taosMemoryCalloc(1, rspSize);
   if (NULL == *pRsp) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
+
   (*pRsp)->useconds = 0;
   (*pRsp)->completed = 1;
   (*pRsp)->precision = 0;
   (*pRsp)->compressed = 0;
   (*pRsp)->compLen = 0;
-  (*pRsp)->numOfRows = htonl(TABLE_TOTAL_COL_NUM(pDesc->pMeta));
-  buildRspData(pDesc->pMeta, (*pRsp)->data);
+  (*pRsp)->numOfRows = htonl(numOfRows);
+  (*pRsp)->numOfCols = htonl(DESCRIBE_RESULT_COLS);
+
+  int32_t len = 0;
+  blockCompressEncode(pBlock, (*pRsp)->data, &len, DESCRIBE_RESULT_COLS, false);
+  ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
+
+  blockDataDestroy(pBlock);
   return TSDB_CODE_SUCCESS;
 }
 

source/libs/command/src/explain.c

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "tdatablock.h"
 #include "commandInt.h"
 #include "plannodes.h"
 #include "query.h"
@@ -657,7 +658,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
 
       if (EXPLAIN_MODE_ANALYZE == ctx->mode) {
         // sort key
-        EXPLAIN_ROW_NEW(level, "Sort Key: ");
+        EXPLAIN_ROW_NEW(level + 1, "Sort Key: ");
         if (pResNode->pExecInfo) {
           for (int32_t i = 0; i < LIST_LENGTH(pSortNode->pSortKeys); ++i) {
             SOrderByExprNode *ptn = nodesListGetNode(pSortNode->pSortKeys, i);
@@ -669,7 +670,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
         QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
 
         // sort method
-        EXPLAIN_ROW_NEW(level, "Sort Method: ");
+        EXPLAIN_ROW_NEW(level + 1, "Sort Method: ");
 
         int32_t           nodeNum = taosArrayGetSize(pResNode->pExecInfo);
         SExplainExecInfo *execInfo = taosArrayGet(pResNode->pExecInfo, 0);
@@ -965,9 +966,32 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
     QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
   }
 
-  int32_t colNum = 1;
-  int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum +
-                    sizeof(int32_t) * rowNum + pCtx->dataSize;
+  SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
+  SColumnInfoData infoData = {0};
+  infoData.info.type  = TSDB_DATA_TYPE_VARCHAR;
+  infoData.info.bytes = TSDB_EXPLAIN_RESULT_ROW_SIZE;
+
+  pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+
+  SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0);
+  pInfoData->hasNull = false;
+  colInfoDataEnsureCapacity(pInfoData, 0, rowNum);
+
+  char buf[1024] = {0};
+  for (int32_t i = 0; i < rowNum; ++i) {
+    SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i);
+    varDataCopy(buf, row->buf);
+    ASSERT(varDataTLen(row->buf) == row->len);
+    colDataAppend(pInfoData, i, buf, false);
+  }
+
+  pBlock->info.numOfCols = 1;
+  pBlock->info.rows = rowNum;
+  pBlock->info.hasVarCol = true;
+
+  int32_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
+
   SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize);
   if (NULL == rsp) {
     qError("malloc SRetrieveTableRsp failed, size:%d", rspSize);
@@ -977,34 +1001,13 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
   rsp->completed = 1;
   rsp->numOfRows = htonl(rowNum);
 
-  // payload length
-  *(int32_t *)rsp->data =
-      sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize;
+  int32_t len = 0;
+  blockCompressEncode(pBlock, rsp->data, &len, pBlock->info.numOfCols, 0);
+  ASSERT(len == rspSize - sizeof(SRetrieveTableRsp));
 
-  // group id
-  *(uint64_t *)(rsp->data + sizeof(int32_t)) = 0;
-
-  // column length
-  int32_t *colLength = (int32_t *)(rsp->data + sizeof(int32_t) + sizeof(uint64_t));
-
-  // varchar column offset segment
-  int32_t *offset = (int32_t *)((char *)colLength + sizeof(int32_t));
-
-  // varchar data real payload
-  char *data = (char *)(offset + rowNum);
-
-  char *start = data;
-  for (int32_t i = 0; i < rowNum; ++i) {
-    SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i);
-    offset[i] = data - start;
-
-    varDataCopy(data, row->buf);
-    ASSERT(varDataTLen(row->buf) == row->len);
-    data += row->len;
-  }
+  rsp->compLen = htonl(len);
 
-  *colLength = htonl(data - start);
-  rsp->compLen = htonl(rspSize);
+  blockDataDestroy(pBlock);
 
   *pRsp = rsp;
   return TSDB_CODE_SUCCESS;

source/libs/executor/inc/executorimpl.h

@@ -353,6 +353,7 @@ typedef struct STagScanInfo {
   int32_t          curPos;
   SReadHandle      readHandle;
   STableListInfo  *pTableList;
+  SNode*           pFilterNode;  // filter info,
 } STagScanInfo;
 
 typedef enum EStreamScanMode {
@@ -687,6 +688,10 @@ typedef struct SSortedMergeOperatorInfo {
   int32_t          numOfResPerPage;
   char**           groupVal;
   SArray          *groupInfo;
+
+  bool hasGroupId;
+  uint64_t groupId;
+  STupleHandle* prefetchedTuple;
 } SSortedMergeOperatorInfo;
 
 typedef struct SSortOperatorInfo {
@@ -699,6 +704,10 @@ typedef struct SSortOperatorInfo {
 
   int64_t      startTs;       // sort start time
   uint64_t     sortElapsed;   // sort elapsed time, time to flush to disk not included.
+
+  STupleHandle *prefetchedTuple;
+  bool  hasGroupId;
+  uint64_t groupId;
 } SSortOperatorInfo;
 
 typedef struct STagFilterOperatorInfo {
@@ -747,7 +756,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scan
 int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz);
 
 void    doSetOperatorCompleted(SOperatorInfo* pOperator);
-void    doFilter(const SNode* pFilterNode, SSDataBlock* pBlock);
+void    doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, bool needFree);
 SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowCellInfoOffset);
 void    relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols);
 void    initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow);
@@ -758,7 +767,7 @@ void    setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData
 SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode);
 SColumn extractColumnFromColumnNode(SColumnNode* pColNode);
 
-SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo);
+SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo, SSortOperatorInfo* pInfo);
 SSDataBlock* loadNextDataBlock(void* param);
 
 void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset);
@@ -777,7 +786,8 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
 SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo);
 
 SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, SReadHandle* pHandle, SArray* groupKyes, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
+SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode,
+                                         STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, SExecTaskInfo* pTaskInfo);
 
 SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo,

source/libs/executor/src/dataDispatcher.c

@@ -68,10 +68,10 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) {
 }
 
 // data format:
-// +----------------+--------------+----------+--------------------------------------+-------------+-----------+-------------+-----------+
-// |SDataCacheEntry | total length | group id | column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | ....
-// |                |  (4 bytes)   |(8 bytes) | sizeof(int32_t) * numOfCols          | actual size |           | actual size |           |
-// +----------------+--------------+----------+--------------------------------------+-------------+-----------+-------------+-----------+
+// +----------------+--------------+----------+--------------------------------------------+--------------------------------------+-------------+-----------+-------------+-----------+
+// |SDataCacheEntry | total length | group id | col1_schema | col2_schema | col3_schema ...| column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | ....
+// |                |  (4 bytes)   |(8 bytes) |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols          | actual size |           | actual size |           |
+// +----------------+--------------+----------+--------------------------------------------+--------------------------------------+-------------+-----------+-------------+-----------+
 // The length of bitmap is decided by number of rows of this data block, and the length of each column data is
 // recorded in the first segment, next to the struct header
 static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) {

source/libs/executor/src/executorimpl.c

@@ -1818,9 +1818,9 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO
   }
 }
 
-static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep);
+static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep, bool needFree);
 
-void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) {
+void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, bool needFree) {
   if (pFilterNode == NULL) {
     return;
   }
@@ -1839,11 +1839,11 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) {
   bool keep = filterExecute(filter, pBlock, &rowRes, NULL, param1.numOfCols);
   filterFreeInfo(filter);
 
-  extractQualifiedTupleByFilterResult(pBlock, rowRes, keep);
+  extractQualifiedTupleByFilterResult(pBlock, rowRes, keep, needFree);
   blockDataUpdateTsWindow(pBlock, 0);
 }
 
-void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep) {
+void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep, bool needFree) {
   if (keep) {
     return;
   }
@@ -1883,8 +1883,20 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR
         ASSERT(pBlock->info.rows == numOfRows);
       }
 
+      SColumnInfoData tmp = *pSrc;
       *pSrc = *pDst;
+      *pDst = tmp;
+
+      if (!needFree) {
+        if (IS_VAR_DATA_TYPE(pDst->info.type)) {  // this elements do not need free
+          pDst->varmeta.offset = NULL;
+        } else {
+          pDst->nullbitmap = NULL;
+        }
+        pDst->pData = NULL;
+      }
     }
+    blockDataDestroy(px);  // fix memory leak
   } else {
     // do nothing
     pBlock->info.rows = 0;
@@ -2559,37 +2571,7 @@ int32_t setDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadIn
       }
     }
 
-    blockDataEnsureCapacity(pBlock, numOfRows);
-
-    int32_t  dataLen = *(int32_t*)pStart;
-    uint64_t groupId = *(uint64_t*)(pStart + sizeof(int32_t));
-    pStart += sizeof(int32_t) + sizeof(uint64_t);
-
-    int32_t* colLen = (int32_t*)(pStart);
-    pStart += sizeof(int32_t) * numOfCols;
-
-    for (int32_t i = 0; i < numOfCols; ++i) {
-      colLen[i] = htonl(colLen[i]);
-      ASSERT(colLen[i] >= 0);
-
-      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
-      if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
-        pColInfoData->varmeta.length = colLen[i];
-        pColInfoData->varmeta.allocLen = colLen[i];
-
-        memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t) * numOfRows);
-        pStart += sizeof(int32_t) * numOfRows;
-
-        pColInfoData->pData = taosMemoryMalloc(colLen[i]);
-      } else {
-        memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows));
-        pStart += BitmapLen(numOfRows);
-      }
-
-      memcpy(pColInfoData->pData, pStart, colLen[i]);
-      pStart += colLen[i];
-    }
-
+    blockCompressDecode(pBlock, numOfCols, numOfRows, pStart);
     // data from mnode
     relocateColumnData(pRes, pColList, pBlock->pDataBlock);
     taosArrayDestroy(pBlock->pDataBlock);
@@ -3121,6 +3103,68 @@ static SSDataBlock* doMerge(SOperatorInfo* pOperator) {
   return (pInfo->binfo.pRes->info.rows > 0) ? pInfo->binfo.pRes : NULL;
 }
 
+SSDataBlock* getSortedMergeBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo,
+                                SSortedMergeOperatorInfo *pInfo) {
+  blockDataCleanup(pDataBlock);
+
+  SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
+  if (p == NULL) {
+    return NULL;
+  }
+
+  blockDataEnsureCapacity(p, capacity);
+
+  while (1) {
+    STupleHandle* pTupleHandle = NULL;
+    if (pInfo->prefetchedTuple == NULL) {
+      pTupleHandle = tsortNextTuple(pHandle);
+    } else {
+      pTupleHandle = pInfo->prefetchedTuple;
+      pInfo->groupId = tsortGetGroupId(pTupleHandle);
+      pInfo->prefetchedTuple = NULL;
+    }
+
+    if (pTupleHandle == NULL) {
+      break;
+    }
+
+    uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle);
+    if (!pInfo->hasGroupId) {
+      pInfo->groupId = tupleGroupId;
+      pInfo->hasGroupId = true;
+      appendOneRowToDataBlock(p, pTupleHandle);
+    } else if (pInfo->groupId == tupleGroupId) {
+      appendOneRowToDataBlock(p, pTupleHandle);
+    } else {
+      pInfo->prefetchedTuple = pTupleHandle;
+      break;
+    }
+
+    if (p->info.rows >= capacity) {
+      break;
+    }
+  }
+
+  if (p->info.rows > 0) {
+    int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
+    for (int32_t i = 0; i < numOfCols; ++i) {
+      SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i);
+      ASSERT(pmInfo->matchType == COL_MATCH_FROM_SLOT_ID);
+
+      SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
+      SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
+      colDataAssign(pDst, pSrc, p->info.rows);
+    }
+
+    pDataBlock->info.rows = p->info.rows;
+    pDataBlock->info.capacity = p->info.rows;
+    pDataBlock->info.groupId = pInfo->groupId;
+  }
+
+  blockDataDestroy(p);
+  return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
+}
+
 static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator) {
   if (pOperator->status == OP_EXEC_DONE) {
     return NULL;
@@ -3129,7 +3173,7 @@ static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator) {
   SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
   SSortedMergeOperatorInfo* pInfo = pOperator->info;
   if (pOperator->status == OP_RES_TO_RETURN) {
-    return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, NULL);
+    return getSortedMergeBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, NULL, pInfo);
   }
 
   int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
@@ -3640,7 +3684,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
       longjmp(pTaskInfo->env, code);
     }
 
-    doFilter(pProjectInfo->pFilterNode, pBlock);
+    doFilter(pProjectInfo->pFilterNode, pBlock, true);
 
     setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, scanFlag, false);
     blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows);
@@ -4522,7 +4566,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       SScanPhysiNode*      pScanPhyNode = (SScanPhysiNode*)pPhyNode;  // simple child table.
       STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
       STimeWindowAggSupp   twSup = {
-            .waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN};
+          .waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN};
       tsdbReaderT pDataReader = NULL;
       if (pHandle->vnode) {
         pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId, pTagCond);

source/libs/executor/src/groupoperator.c

@@ -359,7 +359,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
 
   while(1) {
     doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
-    doFilter(pInfo->pCondition, pRes);
+    doFilter(pInfo->pCondition, pRes, true);
 
     bool hasRemain = hashRemainDataInGroupInfo(&pInfo->groupResInfo);
     if (!hasRemain) {

source/libs/executor/src/scanoperator.c

@@ -267,7 +267,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
   }
 
   int64_t st = taosGetTimestampMs();
-  doFilter(pTableScanInfo->pFilterNode, pBlock);
+  doFilter(pTableScanInfo->pFilterNode, pBlock, false);
 
   int64_t et = taosGetTimestampMs();
   pTableScanInfo->readRecorder.filterTime += (et - st);
@@ -737,8 +737,8 @@ static bool isStateWindow(SStreamBlockScanInfo* pInfo) {
 static bool prepareDataScan(SStreamBlockScanInfo* pInfo) {
   SSDataBlock* pSDB = pInfo->pUpdateRes;
   STimeWindow  win = {
-       .skey = INT64_MIN,
-       .ekey = INT64_MAX,
+      .skey = INT64_MIN,
+      .ekey = INT64_MAX,
   };
   bool needRead = false;
   if (!isStateWindow(pInfo) && pInfo->updateResIndex < pSDB->info.rows) {
@@ -950,7 +950,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
         addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes);
       }
 
-      doFilter(pInfo->pCondition, pInfo->pRes);
+      doFilter(pInfo->pCondition, pInfo->pRes, false);
       blockDataUpdateTsWindow(pInfo->pRes, 0);
       break;
     }
@@ -1722,7 +1722,9 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
   }
 
   pRes->info.rows = count;
-  pOperator->resultInfo.totalRows += count;
+  doFilter(pInfo->pFilterNode, pRes, true);
+
+  pOperator->resultInfo.totalRows += pRes->info.rows;
 
   return (pRes->info.rows == 0) ? NULL : pInfo->pRes;
 }

source/libs/executor/src/sortoperator.c

@@ -42,6 +42,8 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR
 
   pInfo->pSortInfo = pSortInfo;
   pInfo->pColMatchInfo = pColMatchColInfo;
+  pInfo->hasGroupId = false;
+  pInfo->prefetchedTuple = NULL;
   pOperator->name = "SortOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT;
   pOperator->blocking = true;
@@ -81,8 +83,8 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
   pBlock->info.rows += 1;
 }
 
-SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity,
-                                SArray* pColMatchInfo) {
+SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo,
+                                SSortOperatorInfo* pInfo) {
   blockDataCleanup(pDataBlock);
 
   SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
@@ -93,14 +95,33 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
   blockDataEnsureCapacity(p, capacity);
 
   while (1) {
-    STupleHandle* pTupleHandle = tsortNextTuple(pHandle);
+    STupleHandle* pTupleHandle = NULL;
+    if (pInfo->prefetchedTuple == NULL) {
+      pTupleHandle = tsortNextTuple(pHandle);
+    } else {
+      pTupleHandle = pInfo->prefetchedTuple;
+      pInfo->groupId = tsortGetGroupId(pTupleHandle);
+      pInfo->prefetchedTuple = NULL;
+    }
+
     if (pTupleHandle == NULL) {
       break;
     }
 
-    appendOneRowToDataBlock(p, pTupleHandle);
+    uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle);
+    if (!pInfo->hasGroupId) {
+      pInfo->groupId = tupleGroupId;
+      pInfo->hasGroupId = true;
+      appendOneRowToDataBlock(p, pTupleHandle);
+    } else if (pInfo->groupId == tupleGroupId) {
+      appendOneRowToDataBlock(p, pTupleHandle);
+    } else {
+      pInfo->prefetchedTuple = pTupleHandle;
+      break;
+    }
+
     if (p->info.rows >= capacity) {
-      return pDataBlock;
+      break;
     }
   }
 
@@ -117,6 +138,7 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
 
     pDataBlock->info.rows = p->info.rows;
     pDataBlock->info.capacity = p->info.rows;
+    pDataBlock->info.groupId = pInfo->groupId;
   }
 
   blockDataDestroy(p);
@@ -188,8 +210,8 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) {
     longjmp(pTaskInfo->env, code);
   }
 
-  SSDataBlock* pBlock =
-      getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo);
+  SSDataBlock* pBlock = getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity,
+                                           pInfo->pColMatchInfo, pInfo);
 
   if (pBlock != NULL) {
     pOperator->resultInfo.totalRows += pBlock->info.rows;
@@ -230,11 +252,11 @@ typedef struct SMultiwaySortMergeOperatorInfo {
   SArray*      pColMatchInfo;  // for index map from table scan output
 
   SSDataBlock* pInputBlock;
-  int64_t startTs;  // sort start time
+  int64_t      startTs;  // sort start time
 
-  bool  hasGroupId;
-  uint64_t groupId;
-  STupleHandle *prefetchedTuple;
+  bool          hasGroupId;
+  uint64_t      groupId;
+  STupleHandle* prefetchedTuple;
 } SMultiwaySortMergeOperatorInfo;
 
 int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) {
@@ -274,7 +296,7 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) {
 }
 
 SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity,
-                                SArray* pColMatchInfo, SMultiwaySortMergeOperatorInfo* pInfo) {
+                                        SArray* pColMatchInfo, SMultiwaySortMergeOperatorInfo* pInfo) {
   blockDataCleanup(pDataBlock);
 
   SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
@@ -285,12 +307,12 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
   blockDataEnsureCapacity(p, capacity);
 
   while (1) {
-
     STupleHandle* pTupleHandle = NULL;
     if (pInfo->prefetchedTuple == NULL) {
       pTupleHandle = tsortNextTuple(pHandle);
     } else {
       pTupleHandle = pInfo->prefetchedTuple;
+      pInfo->groupId = tsortGetGroupId(pTupleHandle);
       pInfo->prefetchedTuple = NULL;
     }
 
@@ -307,14 +329,12 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
       appendOneRowToDataBlock(p, pTupleHandle);
     } else {
       pInfo->prefetchedTuple = pTupleHandle;
-      pInfo->groupId = tupleGroupId;
       break;
     }
 
     if (p->info.rows >= capacity) {
       break;
     }
-
   }
 
   if (p->info.rows > 0) {
@@ -330,13 +350,13 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
 
     pDataBlock->info.rows = p->info.rows;
     pDataBlock->info.capacity = p->info.rows;
+    pDataBlock->info.groupId = pInfo->groupId;
   }
 
   blockDataDestroy(p);
   return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
 }
 
-
 SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
   if (pOperator->status == OP_EXEC_DONE) {
     return NULL;
@@ -350,12 +370,8 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
     longjmp(pTaskInfo->env, code);
   }
 
-  SSDataBlock* pBlock =
-      getMultiwaySortedBlockData(pInfo->pSortHandle,
-                         pInfo->binfo.pRes,
-                         pOperator->resultInfo.capacity,
-                         pInfo->pColMatchInfo,
-                                 pInfo);
+  SSDataBlock* pBlock = getMultiwaySortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes,
+                                                   pOperator->resultInfo.capacity, pInfo->pColMatchInfo, pInfo);
 
   if (pBlock != NULL) {
     pOperator->resultInfo.totalRows += pBlock->info.rows;
@@ -366,7 +382,7 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
 }
 
 void destroyMultiwaySortMergeOperatorInfo(void* param, int32_t numOfOutput) {
-  SMultiwaySortMergeOperatorInfo * pInfo = (SMultiwaySortMergeOperatorInfo*)param;
+  SMultiwaySortMergeOperatorInfo* pInfo = (SMultiwaySortMergeOperatorInfo*)param;
   pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes);
   pInfo->pInputBlock = blockDataDestroy(pInfo->pInputBlock);
 
@@ -386,9 +402,9 @@ int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrEx
   return TSDB_CODE_SUCCESS;
 }
 
-SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams, SSDataBlock* pInputBlock,
-                                                   SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pColMatchColInfo,
-                                                   SExecTaskInfo* pTaskInfo) {
+SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams,
+                                                   SSDataBlock* pInputBlock, SSDataBlock* pResBlock, SArray* pSortInfo,
+                                                   SArray* pColMatchColInfo, SExecTaskInfo* pTaskInfo) {
   SMultiwaySortMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwaySortMergeOperatorInfo));
   SOperatorInfo*                  pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   int32_t                         rowSize = pResBlock->info.rowSize;
@@ -412,7 +428,8 @@ SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams,
 
   pInfo->bufPageSize = rowSize < 1024 ? 1024 : rowSize * 2;
   pInfo->sortBufSize = pInfo->bufPageSize * 16;
-
+  pInfo->hasGroupId = false;
+  pInfo->prefetchedTuple = NULL;
   pOperator->pTaskInfo = pTaskInfo;
   pOperator->fpSet =
       createOperatorFpSet(doOpenMultiwaySortMergeOperator, doMultiwaySortMerge, NULL, NULL,

source/libs/executor/src/timewindowoperator.c

@@ -1227,7 +1227,10 @@ void doClearWindow(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, char* pData, int
   SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId);
   SResultRowPosition* p1 =
       (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
-  ASSERT(p1);
+  if (!p1) {
+    // window has been closed
+    return;
+  }
   doClearWindowImpl(p1, pSup->pResultBuf, pBinfo, numOfOutput);
 }
 
@@ -2202,12 +2205,12 @@ void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) {
   }
 }
 
-int32_t initBiasicInfo(SOptrBasicInfo* pBasicInfo, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock,
-                       SDiskbasedBuf* pResultBuf) {
+int32_t initBiasicInfo(SOptrBasicInfo* pBasicInfo, SExprInfo* pExprInfo,
+    int32_t numOfCols, SSDataBlock* pResultBlock) {
   pBasicInfo->pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pBasicInfo->rowCellInfoOffset);
   pBasicInfo->pRes = pResultBlock;
   for (int32_t i = 0; i < numOfCols; ++i) {
-    pBasicInfo->pCtx[i].pBuf = pResultBuf;
+    pBasicInfo->pCtx[i].pBuf = NULL;
   }
   return TSDB_CODE_SUCCESS;
 }
@@ -2237,16 +2240,15 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SEx
 
   initResultSizeInfo(pOperator, 4096);
 
-  code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo");
+  code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
-
-  code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock, pInfo->streamAggSup.pResultBuf);
+  pInfo->streamAggSup.resultRowSize = getResultRowSize(pInfo->binfo.pCtx, numOfCols);
+  code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo");
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
-  pInfo->streamAggSup.resultRowSize = getResultRowSize(pInfo->binfo.pCtx, numOfCols);
 
   pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
   if (pInfo->pDummyCtx == NULL) {
@@ -3101,6 +3103,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
   SSDataBlock*                 pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
   int32_t                      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
   SColumnNode*                 pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr;
+  int32_t                      code = TSDB_CODE_OUT_OF_MEMORY;
 
   SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));
   SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
@@ -3121,17 +3124,18 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
       .winMap = NULL,
   };
   initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
-  int32_t code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo");
+
+  code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
 
-  code = initBiasicInfo(&pInfo->binfo, pExprInfo, numOfCols, pResBlock, pInfo->streamAggSup.pResultBuf);
+  pInfo->streamAggSup.resultRowSize = getResultRowSize(pInfo->binfo.pCtx, numOfCols);
+  code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo");
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
 
-  pInfo->streamAggSup.resultRowSize = getResultRowSize(pInfo->binfo.pCtx, numOfCols);
   pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx));
   if (pInfo->pDummyCtx == NULL) {
     goto _error;

source/libs/executor/src/tsort.c

@@ -144,7 +144,9 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf *pBuf, SArray* pAllSource
   (*sourceId) += 1;
 
   int32_t rowSize = blockDataGetSerialRowSize(pSource->src.pBlock);
-  int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(pBlock))/rowSize;  // The value of numOfRows must be greater than 0, which is guaranteed by the previous memory allocation
+
+  // The value of numOfRows must be greater than 0, which is guaranteed by the previous memory allocation
+  int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(pBlock->info.numOfCols))/rowSize;
   ASSERT(numOfRows > 0);
   return blockDataEnsureCapacity(pSource->src.pBlock, numOfRows);
 }
@@ -359,6 +361,11 @@ int32_t msortComparFn(const void *pLeft, const void *pRight, void *param) {
   SSDataBlock* pLeftBlock = pLeftSource->src.pBlock;
   SSDataBlock* pRightBlock = pRightSource->src.pBlock;
 
+  // first sort by block groupId
+  if (pLeftBlock->info.groupId != pRightBlock->info.groupId) {
+    return pLeftBlock->info.groupId < pRightBlock->info.groupId ? -1 : 1;
+  }
+
   for(int32_t i = 0; i < pInfo->size; ++i) {
     SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i);
     SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->slotId);
@@ -521,16 +528,24 @@ static int32_t createInitialSources(SSortHandle* pHandle) {
   if (pHandle->type == SORT_SINGLESOURCE_SORT) {
     SSortSource* source = taosArrayGetP(pHandle->pOrderedSource, 0);
     taosArrayClear(pHandle->pOrderedSource);
-    
+
+    bool hasGroupId = false;
+    SSDataBlock* prefetchedDataBlock = NULL;
+
     while (1) {
-      SSDataBlock* pBlock = pHandle->fetchfp(source->param);
+      SSDataBlock* pBlock = NULL;
+      if (prefetchedDataBlock == NULL) {
+        pBlock = pHandle->fetchfp(source->param);
+      } else {
+        pBlock = prefetchedDataBlock;
+        prefetchedDataBlock = NULL;
+      }
+
       if (pBlock == NULL) {
         break;
       }
 
-      if (pHandle->pDataBlock == NULL) {
-        pHandle->pDataBlock = createOneDataBlock(pBlock, false);
-
+      if (!hasGroupId) {
         // calculate the buffer pages according to the total available buffers.
         int32_t rowSize = blockDataGetRowSize(pBlock);
         if (rowSize * 4 > 4096) {
@@ -542,29 +557,36 @@ static int32_t createInitialSources(SSortHandle* pHandle) {
         // todo!!
         pHandle->numOfPages = 1024;
         sortBufSize = pHandle->numOfPages * pHandle->pageSize;
-      }
 
-      // perform the scalar function calculation before apply the sort
-      if (pHandle->beforeFp != NULL) {
-        pHandle->beforeFp(pBlock, pHandle->param);
+        hasGroupId = true;
+        pHandle->pDataBlock = createOneDataBlock(pBlock, false);
       }
 
-      // todo relocate the columns
-      int32_t code = blockDataMerge(pHandle->pDataBlock, pBlock);
-      if (code != 0) {
-        return code;
-      }
+      if (pHandle->pDataBlock->info.groupId == pBlock->info.groupId) {
+        // perform the scalar function calculation before apply the sort
+        if (pHandle->beforeFp != NULL) {
+          pHandle->beforeFp(pBlock, pHandle->param);
+        }
+        // todo relocate the columns
+        int32_t code = blockDataMerge(pHandle->pDataBlock, pBlock);
+        if (code != 0) {
+          return code;
+        }
 
-      size_t size = blockDataGetSize(pHandle->pDataBlock);
-      if (size > sortBufSize) {
-        // Perform the in-memory sort and then flush data in the buffer into disk.
-        int64_t p = taosGetTimestampUs();
-        blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo);
+        size_t size = blockDataGetSize(pHandle->pDataBlock);
+        if (size > sortBufSize) {
+          // Perform the in-memory sort and then flush data in the buffer into disk.
+          int64_t p = taosGetTimestampUs();
+          blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo);
 
-        int64_t el = taosGetTimestampUs() - p;
-        pHandle->sortElapsed += el;
+          int64_t el = taosGetTimestampUs() - p;
+          pHandle->sortElapsed += el;
 
-        doAddToBuf(pHandle->pDataBlock, pHandle);
+          doAddToBuf(pHandle->pDataBlock, pHandle);
+        }
+      } else {
+        prefetchedDataBlock = pBlock;
+        pHandle->pDataBlock = createOneDataBlock(pBlock, false);
       }
     }
 

source/libs/function/inc/builtinsimpl.h

@@ -23,9 +23,13 @@ extern "C" {
 #include "function.h"
 #include "functionMgt.h"
 
+bool dummyGetEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* UNUSED_PARAM(pEnv));
+bool dummyInit(SqlFunctionCtx* UNUSED_PARAM(pCtx), SResultRowEntryInfo* UNUSED_PARAM(pResultInfo));
+int32_t dummyProcess(SqlFunctionCtx* UNUSED_PARAM(pCtx));
+int32_t dummyFinalize(SqlFunctionCtx* UNUSED_PARAM(pCtx), SSDataBlock* UNUSED_PARAM(pBlock));
+
 bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
-int32_t dummyProcess(SqlFunctionCtx* UNUSED_PARAM(pCtx));
 int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, char* finalResult);
 int32_t combineFunction(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
 
@@ -77,7 +81,11 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 bool getApercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool apercentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t apercentileFunction(SqlFunctionCtx *pCtx);
+int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx);
 int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t apercentilePartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
+int32_t getApercentileMaxSize();
 
 bool getDiffFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool diffFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo);
@@ -98,7 +106,10 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 bool getSpreadFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool spreadFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t spreadFunction(SqlFunctionCtx* pCtx);
+int32_t spreadFunctionMerge(SqlFunctionCtx* pCtx);
 int32_t spreadFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t spreadPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t getSpreadInfoSize();
 
 bool getElapsedFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool elapsedFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
@@ -108,7 +119,10 @@ int32_t elapsedFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
 bool getHistogramFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 bool histogramFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t histogramFunction(SqlFunctionCtx* pCtx);
+int32_t histogramFunctionMerge(SqlFunctionCtx* pCtx);
 int32_t histogramFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t histogramPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
+int32_t getHistogramInfoSize();
 
 bool getHLLFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 int32_t hllFunction(SqlFunctionCtx* pCtx);

source/libs/function/src/builtins.c

@@ -251,6 +251,72 @@ static int32_t translateApercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t translateApercentileImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) {
+  int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
+
+  if (isPartial) {
+    if (2 != numOfParams && 3 != numOfParams) {
+      return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+    // param1
+    SNode* pParamNode1 = nodesListGetNode(pFunc->pParameterList, 1);
+    if (nodeType(pParamNode1) != QUERY_NODE_VALUE) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    SValueNode* pValue = (SValueNode*)pParamNode1;
+    if (pValue->datum.i < 0 || pValue->datum.i > 100) {
+      return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    pValue->notReserved = true;
+
+    uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+    uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type;
+    if (!IS_NUMERIC_TYPE(para1Type) || !IS_INTEGER_TYPE(para2Type)) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    // param2
+    if (3 == numOfParams) {
+      uint8_t para3Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type;
+      if (!IS_VAR_DATA_TYPE(para3Type)) {
+        return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+      }
+
+      SNode* pParamNode2 = nodesListGetNode(pFunc->pParameterList, 2);
+      if (QUERY_NODE_VALUE != nodeType(pParamNode2) || !validateApercentileAlgo((SValueNode*)pParamNode2)) {
+        return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                               "Third parameter algorithm of apercentile must be 'default' or 't-digest'");
+      }
+
+      pValue = (SValueNode*)pParamNode2;
+      pValue->notReserved = true;
+    }
+
+    pFunc->node.resType = (SDataType){.bytes = getApercentileMaxSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
+  } else {
+    if (1 != numOfParams) {
+      return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+    uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+    if (TSDB_DATA_TYPE_BINARY != para1Type) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE};
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t translateApercentilePartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  return translateApercentileImpl(pFunc, pErrBuf, len, true);
+}
+static int32_t translateApercentileMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  return translateApercentileImpl(pFunc, pErrBuf, len, false);
+}
+
 static int32_t translateTbnameColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   // pseudo column do not need to check parameters
   pFunc->node.resType =
@@ -311,6 +377,34 @@ static int32_t translateSpread(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t translateSpreadImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) {
+  if (1 != LIST_LENGTH(pFunc->pParameterList)) {
+    return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+
+  uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+  if (isPartial) {
+    if (!IS_NUMERIC_TYPE(paraType) && TSDB_DATA_TYPE_TIMESTAMP != paraType) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+    pFunc->node.resType = (SDataType){.bytes = getSpreadInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
+  } else {
+    if (TSDB_DATA_TYPE_BINARY != paraType) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+    pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE};
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t translateSpreadPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  return translateSpreadImpl(pFunc, pErrBuf, len, true);
+}
+static int32_t translateSpreadMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  return translateSpreadImpl(pFunc, pErrBuf, len, false);
+}
+
 static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
   if (1 != numOfParams && 2 != numOfParams) {
@@ -409,6 +503,58 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t translateHistogramImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) {
+  int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
+  if (isPartial) {
+    if (4 != numOfParams) {
+      return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+    if (!IS_NUMERIC_TYPE(colType)) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    // param1 ~ param3
+    for (int32_t i = 1; i < numOfParams; ++i) {
+      SNode* pParamNode = nodesListGetNode(pFunc->pParameterList, i);
+      if (QUERY_NODE_VALUE != nodeType(pParamNode)) {
+        return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+      }
+
+      SValueNode* pValue = (SValueNode*)pParamNode;
+
+      pValue->notReserved = true;
+    }
+
+    if (((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type != TSDB_DATA_TYPE_BINARY ||
+        ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type != TSDB_DATA_TYPE_BINARY ||
+        ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 3))->resType.type != TSDB_DATA_TYPE_BIGINT) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    pFunc->node.resType = (SDataType){.bytes = getHistogramInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
+  } else {
+    if (1 != numOfParams) {
+      return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    if (((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type != TSDB_DATA_TYPE_BINARY) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+
+    pFunc->node.resType = (SDataType){.bytes = 512, .type = TSDB_DATA_TYPE_BINARY};
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t translateHistogramPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  return translateHistogramImpl(pFunc, pErrBuf, len, true);
+}
+static int32_t translateHistogramMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  return translateHistogramImpl(pFunc, pErrBuf, len, false);
+}
+
 static int32_t translateHLL(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   if (1 != LIST_LENGTH(pFunc->pParameterList)) {
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
@@ -1056,9 +1202,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .processFunc  = countFunction,
     .finalizeFunc = functionFinalize,
     .invertFunc   = countInvertFunction,
-    .combineFunc = combineFunction,
-    // .pPartialFunc = "count",
-    // .pMergeFunc   = "sum"
+    .combineFunc  = combineFunction,
+    .pPartialFunc = "count",
+    .pMergeFunc   = "sum"
   },
   {
     .name = "sum",
@@ -1071,7 +1217,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .processFunc  = sumFunction,
     .finalizeFunc = functionFinalize,
     .invertFunc   = sumInvertFunction,
-    .combineFunc = sumCombine,
+    .combineFunc  = sumCombine,
+    .pPartialFunc = "sum",
+    .pMergeFunc   = "sum"
   },
   {
     .name = "min",
@@ -1083,7 +1231,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .initFunc     = minmaxFunctionSetup,
     .processFunc  = minFunction,
     .finalizeFunc = minmaxFunctionFinalize,
-    .combineFunc = minCombine
+    .combineFunc  = minCombine,
+    .pPartialFunc = "min",
+    .pMergeFunc   = "min"
   },
   {
     .name = "max",
@@ -1095,7 +1245,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .initFunc     = minmaxFunctionSetup,
     .processFunc  = maxFunction,
     .finalizeFunc = minmaxFunctionFinalize,
-    .combineFunc = maxCombine
+    .combineFunc  = maxCombine,
+    .pPartialFunc = "max",
+    .pMergeFunc   = "max"
   },
   {
     .name = "stddev",
@@ -1141,7 +1293,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getPercentileFuncEnv,
     .initFunc     = percentileFunctionSetup,
     .processFunc  = percentileFunction,
-    .finalizeFunc = percentileFinalize
+    .finalizeFunc = percentileFinalize,
+    .invertFunc   = NULL,
+    .combineFunc  = NULL,
   },
   {
     .name = "apercentile",
@@ -1151,6 +1305,29 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getApercentileFuncEnv,
     .initFunc     = apercentileFunctionSetup,
     .processFunc  = apercentileFunction,
+    .finalizeFunc = apercentileFinalize,
+    .combineFunc  = apercentileCombine,
+    .pPartialFunc = "_apercentile_partial",
+    .pMergeFunc   = "_apercentile_merge"
+  },
+  {
+    .name = "_apercentile_partial",
+    .type = FUNCTION_TYPE_APERCENTILE_PARTIAL,
+    .classification = FUNC_MGT_AGG_FUNC,
+    .translateFunc = translateApercentilePartial,
+    .getEnvFunc   = getApercentileFuncEnv,
+    .initFunc     = apercentileFunctionSetup,
+    .processFunc  = apercentileFunction,
+    .finalizeFunc = apercentilePartialFinalize
+  },
+  {
+    .name = "_apercentile_merge",
+    .type = FUNCTION_TYPE_APERCENTILE_MERGE,
+    .classification = FUNC_MGT_AGG_FUNC,
+    .translateFunc = translateApercentileMerge,
+    .getEnvFunc   = getApercentileFuncEnv,
+    .initFunc     = functionSetup,
+    .processFunc  = apercentileFunctionMerge,
     .finalizeFunc = apercentileFinalize
   },
   {
@@ -1182,6 +1359,30 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getSpreadFuncEnv,
     .initFunc     = spreadFunctionSetup,
     .processFunc  = spreadFunction,
+    .finalizeFunc = spreadFinalize,
+    .pPartialFunc = "_spread_partial",
+    .pMergeFunc   = "_spread_merge"
+  },
+  {
+    .name = "_spread_partial",
+    .type = FUNCTION_TYPE_SPREAD_PARTIAL,
+    .classification = FUNC_MGT_AGG_FUNC,
+    .translateFunc = translateSpreadPartial,
+    .dataRequiredFunc = statisDataRequired,
+    .getEnvFunc   = getSpreadFuncEnv,
+    .initFunc     = spreadFunctionSetup,
+    .processFunc  = spreadFunction,
+    .finalizeFunc = spreadPartialFinalize
+  },
+  {
+    .name = "_spread_merge",
+    .type = FUNCTION_TYPE_SPREAD_MERGE,
+    .classification = FUNC_MGT_AGG_FUNC,
+    .translateFunc = translateSpreadMerge,
+    .dataRequiredFunc = statisDataRequired,
+    .getEnvFunc   = getSpreadFuncEnv,
+    .initFunc     = spreadFunctionSetup,
+    .processFunc  = spreadFunctionMerge,
     .finalizeFunc = spreadFinalize
   },
   {
@@ -1214,7 +1415,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .initFunc     = functionSetup,
     .processFunc  = firstFunction,
     .finalizeFunc = firstLastFinalize,
-    .combineFunc = firstCombine,
+    .combineFunc  = firstCombine,
   },
   {
     .name = "last",
@@ -1245,6 +1446,28 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getHistogramFuncEnv,
     .initFunc     = histogramFunctionSetup,
     .processFunc  = histogramFunction,
+    .finalizeFunc = histogramFinalize,
+    .pPartialFunc = "_histogram_partial",
+    .pMergeFunc   = "_histogram_merge"
+  },
+  {
+    .name = "_histogram_partial",
+    .type = FUNCTION_TYPE_HISTOGRAM_PARTIAL,
+    .classification = FUNC_MGT_AGG_FUNC,
+    .translateFunc = translateHistogramPartial,
+    .getEnvFunc   = getHistogramFuncEnv,
+    .initFunc     = histogramFunctionSetup,
+    .processFunc  = histogramFunction,
+    .finalizeFunc = histogramPartialFinalize
+  },
+  {
+    .name = "_histogram_merge",
+    .type = FUNCTION_TYPE_HISTOGRAM_MERGE,
+    .classification = FUNC_MGT_AGG_FUNC,
+    .translateFunc = translateHistogramMerge,
+    .getEnvFunc   = getHistogramFuncEnv,
+    .initFunc     = functionSetup,
+    .processFunc  = histogramFunctionMerge,
     .finalizeFunc = histogramFinalize
   },
   {

source/libs/index/inc/indexCache.h

@@ -70,7 +70,7 @@ void indexCacheBroadcast(void* cache);
 void indexCacheWait(void* cache);
 
 Iterate* indexCacheIteratorCreate(IndexCache* cache);
-void     indexCacheIteratorDestroy(Iterate* iiter);
+void     idxCacheIteratorDestroy(Iterate* iiter);
 
 int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid);
 
@@ -82,7 +82,7 @@ void indexCacheUnRef(IndexCache* cache);
 
 void indexCacheDebug(IndexCache* cache);
 
-void indexCacheDestroyImm(IndexCache* cache);
+void idxCacheDestroyImm(IndexCache* cache);
 #ifdef __cplusplus
 }
 #endif

source/libs/index/inc/indexComm.h

@@ -26,8 +26,9 @@ extern "C" {
 extern char JSON_COLUMN[];
 extern char JSON_VALUE_DELIM;
 
-char* indexPackJsonData(SIndexTerm* itm);
-char* indexPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip);
+char* idxPackJsonData(SIndexTerm* itm);
+char* idxPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip);
+char* idxPackJsonDataPrefixNoType(SIndexTerm* itm, int32_t* skip);
 
 typedef enum { MATCH, CONTINUE, BREAK } TExeCond;
 
@@ -39,13 +40,14 @@ TExeCond      tDoCompare(__compar_fn_t func, int8_t cmpType, void* a, void* b);
 
 _cache_range_compare indexGetCompare(RangeType ty);
 
-int32_t indexConvertData(void* src, int8_t type, void** dst);
-int32_t indexConvertDataToStr(void* src, int8_t type, void** dst);
+int32_t idxConvertData(void* src, int8_t type, void** dst);
+int32_t idxConvertDataToStr(void* src, int8_t type, void** dst);
 
-int32_t indexGetDataByteLen(int8_t type);
+int32_t idxGetDataByteLen(int8_t type);
 
-char* indexInt2str(int64_t val, char* dst, int radix);
+char* idxInt2str(int64_t val, char* dst, int radix);
 
+int idxUidCompare(const void* a, const void* b);
 #ifdef __cplusplus
 }
 #endif

source/libs/index/inc/indexFst.h

@@ -29,9 +29,9 @@ extern "C" {
 
 #define OUTPUT_PREFIX(a, b) ((a) > (b) ? (b) : (a)
 
-typedef struct Fst             Fst;
-typedef struct FstNode         FstNode;
-typedef struct StreamWithState StreamWithState;
+typedef struct Fst     Fst;
+typedef struct FstNode FstNode;
+typedef struct FStmSt  FStmSt;
 
 typedef enum { Included, Excluded, Unbounded } FstBound;
 
@@ -40,12 +40,12 @@ typedef struct FstBoundWithData {
   FstBound type;
 } FstBoundWithData;
 
-typedef struct FstStreamBuilder {
+typedef struct FStmBuilder {
   Fst*              fst;
-  AutomationCtx*    aut;
+  FAutoCtx*         aut;
   FstBoundWithData* min;
   FstBoundWithData* max;
-} FstStreamBuilder, FstStreamWithStateBuilder;
+} FStmBuilder, FStmStBuilder;
 
 typedef struct FstRange {
   uint64_t start;
@@ -267,17 +267,17 @@ typedef struct Fst {
 Fst* fstCreate(FstSlice* data);
 void fstDestroy(Fst* fst);
 
-bool              fstGet(Fst* fst, FstSlice* b, Output* out);
-FstNode*          fstGetNode(Fst* fst, CompiledAddr);
-FstNode*          fstGetRoot(Fst* fst);
-FstType           fstGetType(Fst* fst);
-CompiledAddr      fstGetRootAddr(Fst* fst);
-Output            fstEmptyFinalOutput(Fst* fst, bool* null);
-FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx);
+bool         fstGet(Fst* fst, FstSlice* b, Output* out);
+FstNode*     fstGetNode(Fst* fst, CompiledAddr);
+FstNode*     fstGetRoot(Fst* fst);
+FstType      fstGetType(Fst* fst);
+CompiledAddr fstGetRootAddr(Fst* fst);
+Output       fstEmptyFinalOutput(Fst* fst, bool* null);
+FStmBuilder* fstSearch(Fst* fst, FAutoCtx* ctx);
 
-FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx);
+FStmStBuilder* fstSearchWithState(Fst* fst, FAutoCtx* ctx);
 // into stream to expand later
-StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb);
+FStmSt* stmBuilderIntoStm(FStmBuilder* sb);
 
 bool fstVerify(Fst* fst);
 
@@ -293,41 +293,40 @@ typedef struct StreamState {
 
 void streamStateDestroy(void* s);
 
-typedef struct StreamWithState {
+typedef struct FStmSt {
   Fst*              fst;
-  AutomationCtx*    aut;
+  FAutoCtx*         aut;
   SArray*           inp;
   FstOutput         emptyOutput;
   SArray*           stack;  // <StreamState>
   FstBoundWithData* endAt;
-} StreamWithState;
+} FStmSt;
 
-typedef struct StreamWithStateResult {
+typedef struct FStmStRslt {
   FstSlice  data;
   FstOutput out;
   void*     state;
-} StreamWithStateResult;
+} FStmStRslt;
 
-StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state);
-void                   swsResultDestroy(StreamWithStateResult* result);
+FStmStRslt* swsResultCreate(FstSlice* data, FstOutput fOut, void* state);
+void        swsResultDestroy(FStmStRslt* result);
 
 typedef void* (*StreamCallback)(void*);
-StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min,
-                                       FstBoundWithData* max);
+FStmSt* stmStCreate(Fst* fst, FAutoCtx* automation, FstBoundWithData* min, FstBoundWithData* max);
 
-void streamWithStateDestroy(StreamWithState* sws);
+void stmStDestroy(FStmSt* sws);
 
-bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min);
+bool stmStSeekMin(FStmSt* sws, FstBoundWithData* min);
 
-StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback);
+FStmStRslt* stmStNextWith(FStmSt* sws, StreamCallback callback);
 
-FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut);
+FStmBuilder* stmBuilderCreate(Fst* fst, FAutoCtx* aut);
 
-void fstStreamBuilderDestroy(FstStreamBuilder* b);
+void stmBuilderDestroy(FStmBuilder* b);
 
 // set up bound range
 // refator later:  to simple code by marco
-void fstStreamBuilderSetRange(FstStreamBuilder* b, FstSlice* val, RangeType type);
+void stmBuilderSetRange(FStmBuilder* b, FstSlice* val, RangeType type);
 
 #ifdef __cplusplus
 }

source/libs/index/inc/indexFstAutomation.h

@@ -22,24 +22,24 @@ extern "C" {
 #include "indexFstUtil.h"
 #include "indexInt.h"
 
-typedef struct AutomationCtx AutomationCtx;
+typedef struct FAutoCtx FAutoCtx;
 
 typedef enum AutomationType { AUTOMATION_ALWAYS, AUTOMATION_PREFIX, AUTMMATION_MATCH } AutomationType;
 
 typedef struct StartWith {
-  AutomationCtx* autoSelf;
+  FAutoCtx* autoSelf;
 } StartWith;
 
 typedef struct Complement {
-  AutomationCtx* autoSelf;
+  FAutoCtx* autoSelf;
 } Complement;
 
 // automation
-typedef struct AutomationCtx {
+typedef struct FAutoCtx {
   AutomationType type;
   void*          stdata;
   char*          data;
-} AutomationCtx;
+} FAutoCtx;
 
 typedef enum ValueType { FST_INT, FST_CHAR, FST_ARRAY } ValueType;
 typedef enum StartWithStateKind { Done, Running } StartWithStateKind;
@@ -60,16 +60,16 @@ StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv);
 void                 startWithStateValueDestroy(void* sv);
 
 typedef struct AutomationFunc {
-  void* (*start)(AutomationCtx* ctx);
-  bool (*isMatch)(AutomationCtx* ctx, void*);
-  bool (*canMatch)(AutomationCtx* ctx, void* data);
-  bool (*willAlwaysMatch)(AutomationCtx* ctx, void* state);
-  void* (*accept)(AutomationCtx* ctx, void* state, uint8_t byte);
-  void* (*acceptEof)(AutomationCtx* ct, void* state);
+  void* (*start)(FAutoCtx* ctx);
+  bool (*isMatch)(FAutoCtx* ctx, void*);
+  bool (*canMatch)(FAutoCtx* ctx, void* data);
+  bool (*willAlwaysMatch)(FAutoCtx* ctx, void* state);
+  void* (*accept)(FAutoCtx* ctx, void* state, uint8_t byte);
+  void* (*acceptEof)(FAutoCtx* ct, void* state);
 } AutomationFunc;
 
-AutomationCtx* automCtxCreate(void* data, AutomationType atype);
-void           automCtxDestroy(AutomationCtx* ctx);
+FAutoCtx* automCtxCreate(void* data, AutomationType atype);
+void      automCtxDestroy(FAutoCtx* ctx);
 
 extern AutomationFunc automFuncs[];
 #ifdef __cplusplus

source/libs/index/inc/indexInt.h

@@ -43,7 +43,7 @@ extern "C" {
 #define indexTrace(...) do { if (idxDebugFlag & DEBUG_TRACE) { taosPrintLog("IDX", DEBUG_TRACE, idxDebugFlag, __VA_ARGS__);} } while (0)
 // clang-format on
 
-typedef enum { LT, LE, GT, GE, CONTAINS } RangeType;
+typedef enum { LT, LE, GT, GE, CONTAINS, EQ } RangeType;
 typedef enum { kTypeValue, kTypeDeletion } STermValueType;
 
 typedef struct SIndexStat {

source/libs/index/src/indexFilter.c

@@ -173,10 +173,8 @@ static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
   param->colId = l->colId;
   param->colValType = l->node.resType.type;
   memcpy(param->dbName, l->dbName, sizeof(l->dbName));
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wformat-overflow"
-  sprintf(param->colName, "%s_%s", l->colName, r->literal);
-#pragma GCC diagnostic pop
+  memcpy(param->colName, r->literal, strlen(r->literal));
+  // sprintf(param->colName, "%s_%s", l->colName, r->literal);
   param->colValType = r->typeData;
   return 0;
   // memcpy(param->colName, l->colName, sizeof(l->colName));
@@ -188,6 +186,9 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
       SIF_ERR_RET(sifGetValueFromNode(node, &param->condValue));
       param->colId = -1;
       param->colValType = (uint8_t)(vn->node.resType.type);
+      if (vn->literal == NULL || strlen(vn->literal) == 0) {
+        return TSDB_CODE_QRY_INVALID_INPUT;
+      }
       memcpy(param->colName, vn->literal, strlen(vn->literal));
       break;
     }
@@ -340,9 +341,9 @@ static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) {
   return NULL;
 }
 static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) {
-  SIndexMetaArg *arg = &output->arg;
-  int            ret = 0;
+  int ret = 0;
 
+  SIndexMetaArg * arg = &output->arg;
   EIndexQueryType qtype = 0;
   SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype));
   if (left->colValType == TSDB_DATA_TYPE_JSON) {
@@ -506,7 +507,6 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) {
   if (nParam <= 1) {
     output->status = SFLT_NOT_INDEX;
     return code;
-    // SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
   }
   if (node->opType == OP_TYPE_JSON_GET_VALUE) {
     return code;
@@ -547,6 +547,8 @@ static int32_t sifExecLogic(SLogicConditionNode *node, SIFCtx *ctx, SIFParam *ou
       } else if (node->condType == LOGIC_COND_TYPE_NOT) {
         // taosArrayAddAll(output->result, params[m].result);
       }
+      taosArraySort(output->result, idxUidCompare);
+      taosArrayRemoveDuplicate(output->result, idxUidCompare, NULL);
     }
   } else {
     for (int32_t m = 0; m < node->pParameterList->length; m++) {

source/libs/index/src/indexJson.c

@@ -22,6 +22,14 @@ int tIndexJsonOpen(SIndexJsonOpts *opts, const char *path, SIndexJson **index) {
 int tIndexJsonPut(SIndexJson *index, SIndexJsonMultiTerm *terms, uint64_t uid) {
   for (int i = 0; i < taosArrayGetSize(terms); i++) {
     SIndexJsonTerm *p = taosArrayGetP(terms, i);
+    if (p->colType == TSDB_DATA_TYPE_BOOL) {
+      p->colType = TSDB_DATA_TYPE_INT;
+    } else if (p->colType == TSDB_DATA_TYPE_VARCHAR || p->colType == TSDB_DATA_TYPE_NCHAR ||
+               p->colType == TSDB_DATA_TYPE_BINARY) {
+      // p->colType = TSDB_DATA_TYPE_NCHAR;
+    } else {
+      p->colType = TSDB_DATA_TYPE_DOUBLE;
+    }
     INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON);
   }
   // handle put
@@ -32,6 +40,14 @@ int tIndexJsonSearch(SIndexJson *index, SIndexJsonMultiTermQuery *tq, SArray *re
   SArray *terms = tq->query;
   for (int i = 0; i < taosArrayGetSize(terms); i++) {
     SIndexJsonTerm *p = taosArrayGetP(terms, i);
+    if (p->colType == TSDB_DATA_TYPE_BOOL) {
+      p->colType = TSDB_DATA_TYPE_INT;
+    } else if (p->colType == TSDB_DATA_TYPE_VARCHAR || p->colType == TSDB_DATA_TYPE_NCHAR ||
+               p->colType == TSDB_DATA_TYPE_BINARY) {
+      // p->colType = TSDB_DATA_TYPE_NCHAR;
+    } else {
+      p->colType = TSDB_DATA_TYPE_DOUBLE;
+    }
     INDEX_TYPE_ADD_EXTERN_TYPE(p->colType, TSDB_DATA_TYPE_JSON);
   }
   // handle search

source/libs/index/test/utilUT.cc

@@ -327,13 +327,13 @@ TEST_F(UtilEnv, testFill) {
   for (int i = 0; i < 1000000; i++) {
     int64_t val = i;
     char    buf[65] = {0};
-    indexInt2str(val, buf, 1);
+    idxInt2str(val, buf, 1);
     EXPECT_EQ(val, taosStr2int64(buf));
   }
   for (int i = 0; i < 1000000; i++) {
     int64_t val = 0 - i;
     char    buf[65] = {0};
-    indexInt2str(val, buf, -1);
+    idxInt2str(val, buf, -1);
     EXPECT_EQ(val, taosStr2int64(buf));
   }
 }