Merge branch '3.0' into feature/3.0_liaohj

cmake/cmake.define

@@ -67,7 +67,13 @@ ELSE ()
     IF (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
         ADD_DEFINITIONS("-D_TD_ARM_")
     ELSE ()
-        ADD_DEFINITIONS("-msse4.2 -mfma")
+        ADD_DEFINITIONS("-msse4.2")
+        IF("${FMA_SUPPORT}" MATCHES "true")
+            MESSAGE(STATUS "turn fma function support on")
+            ADD_DEFINITIONS("-mfma")
+        ELSE ()
+            MESSAGE(STATUS "turn fma function support off")
+        ENDIF()
     ENDIF ()
 
 ENDIF ()

include/common/taosdef.h

@@ -79,11 +79,11 @@ typedef enum {
 } ETsdbSmaType;
 
 typedef enum {
-  TSDB_RSMA_RETENTION_0 = 0,
-  TSDB_RSMA_RETENTION_1 = 1,
-  TSDB_RSMA_RETENTION_2 = 2,
-  TSDB_RSMA_RETENTION_MAX = 3
-} ERSmaRetention;
+  TSDB_RETENTION_L0 = 0,
+  TSDB_RETENTION_L1 = 1,
+  TSDB_RETENTION_L2 = 2,
+  TSDB_RETENTION_MAX = 3
+} ERetentionLevel;
 
 extern char *qtypeStr[];
 

include/common/tdatablock.h

@@ -225,6 +225,9 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData);
 
 void blockDebugShowData(const SArray* dataBlocks);
 
+int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId,
+                                 tb_uid_t uid, tb_uid_t suid);
+
 static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) {
   return blockDataGetSerialMetaSize(pBlock) + blockDataGetSize(pBlock);
 }

include/common/tglobal.h

@@ -121,6 +121,10 @@ extern char     tsCompressor[];
 extern int32_t  tsDiskCfgNum;
 extern SDiskCfg tsDiskCfg[];
 
+// internal 
+extern int32_t tsTransPullupMs;
+extern int32_t tsMaRebalanceMs;
+
 #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
 
 int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile,

include/common/tmsgdef.h

@@ -202,6 +202,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT_RSMA, "vnode-submit-rsma", SSubmitReq, SSubmitRsp)
 
   // sync integration
   TD_DEF_MSG_TYPE(TDMT_VND_SYNC_TIMEOUT, "vnode-sync-timeout", NULL, NULL)

include/libs/index/index.h

@@ -17,6 +17,7 @@
 #define _TD_INDEX_H_
 
 #include "os.h"
+#include "taoserror.h"
 #include "tarray.h"
 
 #ifdef __cplusplus
@@ -41,11 +42,22 @@ typedef enum {
   UPDATE_VALUE,  // update index column value
   ADD_INDEX,     // add index on specify column
   DROP_INDEX,    // drop existed index
-  DROP_SATBLE    // drop stable
+  DROP_SATBLE,   // drop stable
+  DEFAULT        // query
 } SIndexOperOnColumn;
 
-typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType;
-typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = 3, QUERY_RANGE = 4 } EIndexQueryType;
+typedef enum { MUST = 0, SHOULD, NOT } EIndexOperatorType;
+typedef enum {
+  QUERY_TERM = 0,
+  QUERY_PREFIX,
+  QUERY_SUFFIX,
+  QUERY_REGEX,
+  QUERY_LESS_THAN,
+  QUERY_LESS_EQUAL,
+  QUERY_GREATER_THAN,
+  QUERY_GREATER_EQUAL,
+  QUERY_RANGE
+} EIndexQueryType;
 
 /*
  * create multi query
@@ -166,8 +178,8 @@ void        indexOptsDestroy(SIndexOpts* opts);
  * @param:
  */
 
-SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, const char* colName,
-                            int32_t nColName, const char* colVal, int32_t nColVal);
+SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn operType, int8_t qType, uint8_t colType,
+                            const char* colName, int32_t nColName, const char* colVal, int32_t nColVal);
 void        indexTermDestroy(SIndexTerm* p);
 
 /*

include/libs/parser/parser.h

@@ -20,8 +20,8 @@
 extern "C" {
 #endif
 
-#include "querynodes.h"
 #include "query.h"
+#include "querynodes.h"
 
 typedef struct SStmtCallback {
   TAOS_STMT* pStmt;
@@ -34,24 +34,26 @@ typedef struct SStmtCallback {
 typedef struct SParseContext {
   uint64_t         requestId;
   int32_t          acctId;
-  const char      *db;
+  const char*      db;
   bool             topicQuery;
-  void            *pTransporter;
+  void*            pTransporter;
   SEpSet           mgmtEpSet;
-  const char      *pSql;           // sql string
-  size_t           sqlLen;         // length of the sql string
-  char            *pMsg;           // extended error message if exists to help identifying the problem in sql statement.
-  int32_t          msgLen;         // max length of the msg
-  struct SCatalog *pCatalog;
-  SStmtCallback   *pStmtCb;
+  const char*      pSql;    // sql string
+  size_t           sqlLen;  // length of the sql string
+  char*            pMsg;    // extended error message if exists to help identifying the problem in sql statement.
+  int32_t          msgLen;  // max length of the msg
+  struct SCatalog* pCatalog;
+  SStmtCallback*   pStmtCb;
+  const char*      pUser;
+  bool             isSuperUser;
 } SParseContext;
 
 typedef struct SCmdMsgInfo {
   int16_t msgType;
-  SEpSet epSet;
-  void* pMsg;
+  SEpSet  epSet;
+  void*   pMsg;
   int32_t msgLen;
-  void* pExtension;  // todo remove it soon
+  void*   pExtension;  // todo remove it soon
 } SCmdMsgInfo;
 
 typedef enum EQueryExecMode {
@@ -63,21 +65,21 @@ typedef enum EQueryExecMode {
 
 typedef struct SQuery {
   EQueryExecMode execMode;
-  bool haveResultSet;
-  SNode* pRoot;
-  int32_t numOfResCols;
-  SSchema* pResSchema;
-  int8_t   precision;
-  SCmdMsgInfo* pCmdMsg;
-  int32_t msgType;
-  SArray* pDbList;
-  SArray* pTableList;
-  bool showRewrite;
-  int32_t placeholderNum;
+  bool           haveResultSet;
+  SNode*         pRoot;
+  int32_t        numOfResCols;
+  SSchema*       pResSchema;
+  int8_t         precision;
+  SCmdMsgInfo*   pCmdMsg;
+  int32_t        msgType;
+  SArray*        pDbList;
+  SArray*        pTableList;
+  bool           showRewrite;
+  int32_t        placeholderNum;
 } SQuery;
 
 int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery);
-bool isInsertSql(const char* pStr, size_t length);
+bool    isInsertSql(const char* pStr, size_t length);
 
 void qDestroyQuery(SQuery* pQueryNode);
 
@@ -89,14 +91,16 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc);
 void    qFreeStmtDataBlock(void* pDataBlock);
 int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc);
 void    qDestroyStmtDataBlock(void* pBlock);
-int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen);
-int32_t qBindStmtSingleColValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum);
-int32_t qBuildStmtColFields(void *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields);
-int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields);
-int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen);
-void destroyBoundColumnInfo(void* pBoundInfo);
-int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen);
-
+int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen);
+int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx,
+                                int32_t rowNum);
+int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD** fields);
+int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD** fields);
+int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, SName* pName, TAOS_MULTI_BIND* bind,
+                           char* msgBuf, int32_t msgBufLen);
+void    destroyBoundColumnInfo(void* pBoundInfo);
+int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf,
+                     int32_t msgBufLen);
 
 #ifdef __cplusplus
 }

include/util/taoserror.h

@@ -264,7 +264,8 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_TRANS_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03D0)
 #define TSDB_CODE_MND_TRANS_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x03D1)
 #define TSDB_CODE_MND_TRANS_INVALID_STAGE       TAOS_DEF_ERROR_CODE(0, 0x03D2)
-#define TSDB_CODE_MND_TRANS_CANT_PARALLEL       TAOS_DEF_ERROR_CODE(0, 0x03D4)
+#define TSDB_CODE_MND_TRANS_CONFLICT            TAOS_DEF_ERROR_CODE(0, 0x03D3)
+#define TSDB_CODE_MND_TRANS_UNKNOW_ERROR        TAOS_DEF_ERROR_CODE(0, 0x03D4)
 
 // mnode-mq
 #define TSDB_CODE_MND_TOPIC_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03E0)

include/util/tskiplist2.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef _TD_UTIL_SKIPLIST2_H_
+#define _TD_UTIL_SKIPLIST2_H_
+
+#include "os.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define SL_MAX_LEVEL 15
+
+typedef struct SSkipList2 SSkipList2;
+typedef struct SSLCursor  SSLCursor;
+typedef struct SSLCfg     SSLCfg;
+typedef struct SSLNode    SSLNode;
+
+typedef int32_t (*tslCmprFn)(const void *pKey1, int32_t nKey1, const void *pKey2, int32_t nKey2);
+
+// SSkipList2
+int32_t slOpen(const SSLCfg *pCfg, SSkipList2 **ppSl);
+int32_t slClose(SSkipList2 *pSl);
+int32_t slClear(SSkipList2 *pSl);
+
+// SSLCursor
+int32_t slcOpen(SSkipList2 *pSl, SSLCursor *pSlc);
+int32_t slcClose(SSLCursor *pSlc);
+int32_t slcMoveTo(SSLCursor *pSlc, const void *pKey, int32_t nKey);
+int32_t slcMoveToNext(SSLCursor *pSlc);
+int32_t slcMoveToPrev(SSLCursor *pSlc);
+int32_t slcMoveToFirst(SSLCursor *pSlc);
+int32_t slcMoveToLast(SSLCursor *pSlc);
+int32_t slcPut(SSLCursor *pSlc, const void *pKey, int32_t nKey, const void *pData, int32_t nData);
+int32_t slcGet(SSLCursor *pSlc, const void **ppKey, int32_t *nKey, const void **ppData, int32_t *nData);
+int32_t slcDrop(SSLCursor *pSlc);
+
+// struct
+struct SSLCfg {
+  int8_t    maxLevel;
+  int32_t   nKey;
+  int32_t   nData;
+  tslCmprFn cmprFn;
+  void     *pPool;
+  void *(*xMalloc)(void *, int32_t size);
+  void (*xFree)(void *, void *);
+};
+
+struct SSLCursor {
+  SSkipList2 *pSl;
+  SSLNode   **forwards[SL_MAX_LEVEL];
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_UTIL_SKIPLIST2_H_*/
\ No newline at end of file

source/client/src/clientStmt.c

@@ -649,6 +649,19 @@ int stmtGetParamNum(TAOS_STMT *stmt, int *nums) {
 
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS));
 
+  if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) {
+    pStmt->bInfo.needParse = false;
+  }
+
+  if (pStmt->exec.pRequest && STMT_TYPE_QUERY == pStmt->sql.type && pStmt->sql.runTimes) {
+    taos_free_result(pStmt->exec.pRequest);
+    pStmt->exec.pRequest = NULL;
+  }
+  
+  if (NULL == pStmt->exec.pRequest) {
+    STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
+  }
+
   if (pStmt->bInfo.needParse) {
     STMT_ERR_RET(stmtParseSql(pStmt));
   }
@@ -658,8 +671,11 @@ int stmtGetParamNum(TAOS_STMT *stmt, int *nums) {
       STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList));
       pStmt->sql.pQueryPlan = pStmt->exec.pRequest->body.pDag;
       pStmt->exec.pRequest->body.pDag = NULL;
+      STMT_ERR_RET(stmtBackupQueryFields(pStmt));
+    } else {
+      STMT_ERR_RET(stmtRestoreQueryFields(pStmt));
     }
-
+    
     *nums = taosArrayGetSize(pStmt->sql.pQueryPlan->pPlaceholderValues);
   } else {
     STMT_ERR_RET(stmtFetchColFields(stmt, nums, NULL));

source/common/src/tdatablock.c

@@ -187,17 +187,17 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c
   int32_t i = 0;
 
   uint8_t* start = (uint8_t*)&pColumnInfoData->nullbitmap[BitmapLen(numOfRow1)];
-  int32_t overCount = BitmapLen(total) - BitmapLen(numOfRow1);
-  while (i < len) { // size limit of pSource->nullbitmap
+  int32_t  overCount = BitmapLen(total) - BitmapLen(numOfRow1);
+  while (i < len) {  // size limit of pSource->nullbitmap
     if (i >= 1) {
-      start[i - 1] |= (p[i] >> remindBits); //copy remind bits
+      start[i - 1] |= (p[i] >> remindBits);  // copy remind bits
     }
 
-    if (i >= overCount) { // size limit of pColumnInfoData->nullbitmap
+    if (i >= overCount) {  // size limit of pColumnInfoData->nullbitmap
       return;
     }
 
-    start[i] |= (p[i] << shiftBits);  //copy shift bits
+    start[i] |= (p[i] << shiftBits);  // copy shift bits
     i += 1;
   }
 }
@@ -453,7 +453,6 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd
   // all fit in
   *stopIndex = numOfRows - 1;
   return TSDB_CODE_SUCCESS;
-
 }
 
 SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount) {
@@ -558,7 +557,7 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) {
 
     if (IS_VAR_DATA_TYPE(pCol->info.type)) {
       size_t metaSize = pBlock->info.rows * sizeof(int32_t);
-      char* tmp = taosMemoryRealloc(pCol->varmeta.offset, metaSize);    // preview calloc is too small
+      char*  tmp = taosMemoryRealloc(pCol->varmeta.offset, metaSize);  // preview calloc is too small
       if (tmp == NULL) {
         return TSDB_CODE_OUT_OF_MEMORY;
       }
@@ -940,8 +939,9 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
   copyBackToBlock(pDataBlock, pCols);
   int64_t p4 = taosGetTimestampUs();
 
-  uDebug("blockDataSort complex sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1,
-         p3 - p2, p4 - p3, rows);
+  uDebug("blockDataSort complex sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64
+         ", rows:%d\n",
+         p1 - p0, p2 - p1, p3 - p2, p4 - p3, rows);
   destroyTupleIndex(index);
 
   return TSDB_CODE_SUCCESS;
@@ -1178,7 +1178,7 @@ void* blockDataDestroy(SSDataBlock* pBlock) {
 }
 
 SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
-  if(pDataBlock == NULL){
+  if (pDataBlock == NULL) {
     return NULL;
   }
 
@@ -1189,7 +1189,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
 
   pBlock->info.numOfCols = numOfCols;
   pBlock->info.hasVarCol = pDataBlock->info.hasVarCol;
-  pBlock->info.rowSize   = pDataBlock->info.rows;
+  pBlock->info.rowSize = pDataBlock->info.rows;
 
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData  colInfo = {0};
@@ -1219,7 +1219,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
 }
 
 size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) {
-  return (int32_t) ((pageSize - blockDataGetSerialMetaSize(pBlock))/ blockDataGetSerialRowSize(pBlock));
+  return (int32_t)((pageSize - blockDataGetSerialMetaSize(pBlock)) / blockDataGetSerialRowSize(pBlock));
 }
 
 void colDataDestroy(SColumnInfoData* pColData) {
@@ -1236,14 +1236,14 @@ static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) {
   int32_t len = BitmapLen(total);
 
   int32_t newLen = BitmapLen(total - n);
-  if (n%8 == 0) {
-    memmove(nullBitmap, nullBitmap + n/8, newLen);
+  if (n % 8 == 0) {
+    memmove(nullBitmap, nullBitmap + n / 8, newLen);
   } else {
     int32_t tail = n % 8;
     int32_t i = 0;
 
-    uint8_t* p = (uint8_t*) nullBitmap;
-    while(i < len) {
+    uint8_t* p = (uint8_t*)nullBitmap;
+    while (i < len) {
       uint8_t v = p[i];
 
       p[i] = 0;
@@ -1270,7 +1270,7 @@ static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_
   }
 }
 
-int32_t blockDataTrimFirstNRows(SSDataBlock *pBlock, size_t n) {
+int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) {
   if (n == 0) {
     return TSDB_CODE_SUCCESS;
   }
@@ -1278,7 +1278,7 @@ int32_t blockDataTrimFirstNRows(SSDataBlock *pBlock, size_t n) {
   if (pBlock->info.rows <= n) {
     blockDataCleanup(pBlock);
   } else {
-    for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
+    for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
       SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
       colDataTrimFirstNRows(pColInfoData, n, pBlock->info.rows);
     }
@@ -1464,3 +1464,128 @@ void blockDebugShowData(const SArray* dataBlocks) {
   }
 }
 
+/**
+ * @brief TODO: Assume that the final generated result it less than 3M
+ *
+ * @param pReq
+ * @param pDataBlocks
+ * @param vgId
+ * @param uid set as parameter temporarily // TODO: remove this parameter, and the executor should set uid in
+ * SDataBlock->info.uid
+ * @param suid  // TODO: check with Liao whether suid response is reasonable
+ * 
+ * TODO: colId should be set
+ */
+int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema *pTSchema, int32_t vgId, tb_uid_t uid,
+                                        tb_uid_t suid) {
+  int32_t sz = taosArrayGetSize(pDataBlocks);
+  int32_t bufSize = sizeof(SSubmitReq);
+  for (int32_t i = 0; i < sz; ++i) {
+    SDataBlockInfo* pBlkInfo = &((SSDataBlock*)taosArrayGet(pDataBlocks, i))->info;
+    bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(pBlkInfo->numOfCols));
+    bufSize += sizeof(SSubmitBlk);
+  }
+
+  ASSERT(bufSize < 3 * 1024 * 1024);
+
+  *pReq = taosMemoryCalloc(1, bufSize);
+  if(!(*pReq)) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return TSDB_CODE_FAILED;
+  }
+  void* pDataBuf = *pReq;
+
+  int32_t msgLen = sizeof(SSubmitReq);
+  int32_t     numOfBlks = 0;
+  SRowBuilder rb = {0};
+  tdSRowInit(&rb, 0); // TODO: use the latest version
+
+  for (int32_t i = 0; i < sz; ++i) {
+    SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i);
+    int32_t      colNum = pDataBlock->info.numOfCols;
+    int32_t      rows = pDataBlock->info.rows;
+    int32_t      rowSize = pDataBlock->info.rowSize;
+    int64_t      groupId = pDataBlock->info.groupId;
+    
+    if(rb.nCols != colNum) {
+      tdSRowSetTpInfo(&rb, colNum, pTSchema->flen);
+    }
+
+    SSubmitBlk* pSubmitBlk = POINTER_SHIFT(pDataBuf, msgLen);
+    pSubmitBlk->suid = suid;
+    pSubmitBlk->uid = uid;
+    pSubmitBlk->numOfRows = rows;
+
+    ++numOfBlks;
+
+    msgLen += sizeof(SSubmitBlk);
+    int32_t dataLen = 0;
+    for (int32_t j = 0; j < rows; ++j) {     // iterate by row
+      tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen)); // set row buf
+      printf("|");
+      bool  isStartKey = false;
+      for (int32_t k = 0; k < colNum; ++k) {  // iterate by column
+        SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
+        void*            var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
+        switch (pColInfoData->info.type) {
+          case TSDB_DATA_TYPE_TIMESTAMP:
+            if (!isStartKey) {
+              isStartKey = true;
+              tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, 0, 0);
+            } else {
+              tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, 8, k);
+              break;
+            }
+            break;
+          case TSDB_DATA_TYPE_NCHAR: {
+            tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_NCHAR, TD_VTYPE_NORM, var, true, 8, k);
+            break;
+          }
+          case TSDB_DATA_TYPE_VARCHAR: {  // TSDB_DATA_TYPE_BINARY
+            tdAppendColValToRow(&rb, 2, TSDB_DATA_TYPE_VARCHAR, TD_VTYPE_NORM, var, true, 8, k);
+            break;
+          }
+          case TSDB_DATA_TYPE_VARBINARY:
+          case TSDB_DATA_TYPE_DECIMAL:
+          case TSDB_DATA_TYPE_BLOB:
+          case TSDB_DATA_TYPE_MEDIUMBLOB:
+            printf("the column type %" PRIi16 " is defined but not implemented yet\n", pColInfoData->info.type);
+            TASSERT(0);
+            break;
+          default:
+            if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) {
+              tdAppendColValToRow(&rb, 2, pColInfoData->info.type, TD_VTYPE_NORM, var, true, 8, k);
+            } else {
+              printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type);
+              TASSERT(0);
+            }
+            break;
+        }
+      }
+      dataLen += TD_ROW_LEN(rb.pBuf);
+    }
+    pSubmitBlk->dataLen = dataLen;
+    msgLen += pSubmitBlk->dataLen;
+  }
+
+  (*pReq)->length = msgLen;
+
+  (*pReq)->header.vgId = htonl(vgId);
+  (*pReq)->header.contLen = htonl(msgLen);
+  (*pReq)->length = (*pReq)->header.contLen;
+  (*pReq)->numOfBlocks = htonl(numOfBlks);
+  SSubmitBlk* blk = (SSubmitBlk*)((*pReq) + 1);
+  while (numOfBlks--) {
+    int32_t dataLen = blk->dataLen;
+    blk->uid = htobe64(blk->uid);
+    blk->suid = htobe64(blk->suid);
+    blk->padding = htonl(blk->padding);
+    blk->sversion = htonl(blk->sversion);
+    blk->dataLen = htonl(blk->dataLen);
+    blk->schemaLen = htonl(blk->schemaLen);
+    blk->numOfRows = htons(blk->numOfRows);
+    blk = (SSubmitBlk*)(blk->data + dataLen);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}

source/common/src/tglobal.c

@@ -169,6 +169,10 @@ uint32_t tsMaxRange = 500;                      // max range
 uint32_t tsCurRange = 100;                      // range
 char     tsCompressor[32] = "ZSTD_COMPRESSOR";  // ZSTD_COMPRESSOR or GZIP_COMPRESSOR
 
+// internal
+int32_t tsTransPullupMs = 6000;
+int32_t tsMaRebalanceMs = 2000;
+
 void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) {
   tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN);
   tsDiskCfg[index].level = level;
@@ -220,7 +224,8 @@ struct SConfig *taosGetCfg() {
   return tsCfg;
 }
 
-static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, char *apolloUrl) {
+static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile,
+                           char *apolloUrl) {
   char cfgDir[PATH_MAX] = {0};
   char cfgFile[PATH_MAX + 100] = {0};
 
@@ -296,15 +301,10 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) {
 static int32_t taosAddClientCfg(SConfig *pCfg) {
   char    defaultFqdn[TSDB_FQDN_LEN] = {0};
   int32_t defaultServerPort = 6030;
-  char    defaultFirstEp[TSDB_EP_LEN] = {0};
-  char    defaultSecondEp[TSDB_EP_LEN] = {0};
-
   if (taosGetFqdn(defaultFqdn) != 0) return -1;
-  snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort);
-  snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort);
 
-  if (cfgAddString(pCfg, "firstEp", defaultFirstEp, 1) != 0) return -1;
-  if (cfgAddString(pCfg, "secondEp", defaultSecondEp, 1) != 0) return -1;
+  if (cfgAddString(pCfg, "firstEp", "", 1) != 0) return -1;
+  if (cfgAddString(pCfg, "secondEp", "", 1) != 0) return -1;
   if (cfgAddString(pCfg, "fqdn", defaultFqdn, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, 1) != 0) return -1;
   if (cfgAddDir(pCfg, "tempDir", tsTempDir, 1) != 0) return -1;
@@ -474,15 +474,18 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
   tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32;
   snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort);
 
+  char defaultFirstEp[TSDB_EP_LEN] = {0};
+  snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort);
+
   SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp");
   SEp          firstEp = {0};
-  taosGetFqdnPortFromEp(pFirstEpItem->str, &firstEp);
+  taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp);
   snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port);
   cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype);
 
   SConfigItem *pSecondpItem = cfgGetItem(pCfg, "secondEp");
   SEp          secondEp = {0};
-  taosGetFqdnPortFromEp(pSecondpItem->str, &secondEp);
+  taosGetFqdnPortFromEp(strlen(pSecondpItem->str) == 0 ? defaultFirstEp : pSecondpItem->str, &secondEp);
   snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port);
   cfgSetItem(pCfg, "secondEp", tsSecond, pSecondpItem->stype);
 
@@ -579,8 +582,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   return 0;
 }
 
-int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile,
-                      char *apolloUrl, SArray *pArgs, bool tsc) {
+int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd,
+                      const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) {
   osDefaultInit();
 
   SConfig *pCfg = cfgInit();
@@ -632,7 +635,24 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
   return 0;
 }
 
-int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) {
+static int32_t taosCheckGlobalCfg() {
+  uint32_t ipv4 = taosGetIpv4FromFqdn(tsLocalFqdn);
+  if (ipv4 == 0xffffffff) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    uError("failed to get ip from fqdn:%s since %s, dnode can not be initialized", tsLocalFqdn, terrstr());
+    return -1;
+  }
+
+  if (tsServerPort <= 0) {
+    uError("invalid server port:%u, dnode can not be initialized", tsServerPort);
+    return -1;
+  }
+
+  return 0;
+}
+
+int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs,
+                    bool tsc) {
   if (tsCfg != NULL) return 0;
   tsCfg = cfgInit();
 
@@ -670,6 +690,11 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
   taosSetSystemCfg(tsCfg);
 
   cfgDumpCfg(tsCfg, tsc, false);
+
+  if (taosCheckGlobalCfg() != 0) {
+    return -1;
+  }
+
   return 0;
 }
 

source/common/src/tmsgcb.c

@@ -15,37 +15,83 @@
 
 #define _DEFAULT_SOURCE
 #include "tmsgcb.h"
+#include "taoserror.h"
 
 static SMsgCb tsDefaultMsgCb;
 
 void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb) { tsDefaultMsgCb = *pMsgCb; }
 
 int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq) {
-  return (*pMsgCb->queueFps[qtype])(pMsgCb->pWrapper, pReq);
+  PutToQueueFp fp = pMsgCb->queueFps[qtype];
+  if (fp != NULL) {
+    return (*fp)(pMsgCb->pWrapper, pReq);
+  } else {
+    terrno = TSDB_CODE_INVALID_PTR;
+    return -1;
+  }
 }
 
 int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype) {
-  return (*pMsgCb->qsizeFp)(pMsgCb->pWrapper, vgId, qtype);
+  GetQueueSizeFp fp = pMsgCb->qsizeFp;
+  if (fp != NULL) {
+    return (*fp)(pMsgCb->pWrapper, vgId, qtype);
+  } else {
+    terrno = TSDB_CODE_INVALID_PTR;
+    return -1;
+  }
 }
 
 int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq) {
-  return (*pMsgCb->sendReqFp)(pMsgCb->pWrapper, epSet, pReq);
+  SendReqFp fp = pMsgCb->sendReqFp;
+  if (fp != NULL) {
+    return (*fp)(pMsgCb->pWrapper, epSet, pReq);
+  } else {
+    terrno = TSDB_CODE_INVALID_PTR;
+    return -1;
+  }
 }
 
-void tmsgSendRsp(const SRpcMsg* pRsp) { return (*tsDefaultMsgCb.sendRspFp)(tsDefaultMsgCb.pWrapper, pRsp); }
+void tmsgSendRsp(const SRpcMsg* pRsp) {
+  SendRspFp fp = tsDefaultMsgCb.sendRspFp;
+  if (fp != NULL) {
+    return (*fp)(tsDefaultMsgCb.pWrapper, pRsp);
+  } else {
+    terrno = TSDB_CODE_INVALID_PTR;
+  }
+}
 
 void tmsgSendRedirectRsp(const SRpcMsg* pRsp, const SEpSet* pNewEpSet) {
-  return (*tsDefaultMsgCb.sendRedirectRspFp)(tsDefaultMsgCb.pWrapper, pRsp, pNewEpSet);
+  SendRedirectRspFp fp = tsDefaultMsgCb.sendRedirectRspFp;
+  if (fp != NULL) {
+    (*fp)(tsDefaultMsgCb.pWrapper, pRsp, pNewEpSet);
+  } else {
+    terrno = TSDB_CODE_INVALID_PTR;
+  }
 }
 
 void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg) {
-  (*pMsgCb->registerBrokenLinkArgFp)(pMsgCb->pWrapper, pMsg);
+  RegisterBrokenLinkArgFp fp = pMsgCb->registerBrokenLinkArgFp;
+  if (fp != NULL) {
+    (*fp)(pMsgCb->pWrapper, pMsg);
+  } else {
+    terrno = TSDB_CODE_INVALID_PTR;
+  }
 }
 
 void tmsgReleaseHandle(void* handle, int8_t type) {
-  (*tsDefaultMsgCb.releaseHandleFp)(tsDefaultMsgCb.pWrapper, handle, type);
+  ReleaseHandleFp fp = tsDefaultMsgCb.releaseHandleFp;
+  if (fp != NULL) {
+    (*fp)(tsDefaultMsgCb.pWrapper, handle, type);
+  } else {
+    terrno = TSDB_CODE_INVALID_PTR;
+  }
 }
 
 void tmsgReportStartup(const char* name, const char* desc) {
-  (*tsDefaultMsgCb.reportStartupFp)(tsDefaultMsgCb.pWrapper, name, desc);
+  ReportStartup fp = tsDefaultMsgCb.reportStartupFp;
+  if (fp != NULL && tsDefaultMsgCb.pWrapper != NULL) {
+    (*fp)(tsDefaultMsgCb.pWrapper, name, desc);
+  } else {
+    terrno = TSDB_CODE_INVALID_PTR;
+  }
 }
\ No newline at end of file

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -284,6 +284,7 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SUBMIT_RSMA, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_CONSUME, vmProcessFetchMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, vmProcessWriteMsg, DEFAULT_HANDLE);

source/dnode/mnode/impl/inc/mndDef.h

@@ -57,11 +57,11 @@ typedef enum {
   TRN_STAGE_PREPARE = 0,
   TRN_STAGE_REDO_LOG = 1,
   TRN_STAGE_REDO_ACTION = 2,
-  TRN_STAGE_COMMIT = 3,
-  TRN_STAGE_COMMIT_LOG = 4,
-  TRN_STAGE_UNDO_ACTION = 5,
-  TRN_STAGE_UNDO_LOG = 6,
-  TRN_STAGE_ROLLBACK = 7,
+  TRN_STAGE_ROLLBACK = 3,
+  TRN_STAGE_UNDO_ACTION = 4,
+  TRN_STAGE_UNDO_LOG = 5,
+  TRN_STAGE_COMMIT = 6,
+  TRN_STAGE_COMMIT_LOG = 7,
   TRN_STAGE_FINISHED = 8
 } ETrnStage;
 
@@ -72,6 +72,7 @@ typedef enum {
   TRN_TYPE_DROP_USER = 1003,
   TRN_TYPE_CREATE_FUNC = 1004,
   TRN_TYPE_DROP_FUNC = 1005,
+  
   TRN_TYPE_CREATE_SNODE = 1006,
   TRN_TYPE_DROP_SNODE = 1007,
   TRN_TYPE_CREATE_QNODE = 1008,
@@ -91,10 +92,12 @@ typedef enum {
   TRN_TYPE_CONSUMER_LOST = 1022,
   TRN_TYPE_CONSUMER_RECOVER = 1023,
   TRN_TYPE_BASIC_SCOPE_END,
+
   TRN_TYPE_GLOBAL_SCOPE = 2000,
   TRN_TYPE_CREATE_DNODE = 2001,
   TRN_TYPE_DROP_DNODE = 2002,
   TRN_TYPE_GLOBAL_SCOPE_END,
+
   TRN_TYPE_DB_SCOPE = 3000,
   TRN_TYPE_CREATE_DB = 3001,
   TRN_TYPE_ALTER_DB = 3002,
@@ -102,6 +105,7 @@ typedef enum {
   TRN_TYPE_SPLIT_VGROUP = 3004,
   TRN_TYPE_MERGE_VGROUP = 3015,
   TRN_TYPE_DB_SCOPE_END,
+
   TRN_TYPE_STB_SCOPE = 4000,
   TRN_TYPE_CREATE_STB = 4001,
   TRN_TYPE_ALTER_STB = 4002,
@@ -131,7 +135,7 @@ typedef struct {
   int32_t    id;
   ETrnStage  stage;
   ETrnPolicy policy;
-  ETrnType   transType;
+  ETrnType   type;
   int32_t    code;
   int32_t    failedTimes;
   void*      rpcHandle;

source/dnode/mnode/impl/inc/mndTrans.h

@@ -44,6 +44,8 @@ typedef void (*TransCbFp)(SMnode *pMnode, void *param, int32_t paramLen);
 
 int32_t mndInitTrans(SMnode *pMnode);
 void    mndCleanupTrans(SMnode *pMnode);
+STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId);
+void    mndReleaseTrans(SMnode *pMnode, STrans *pTrans);
 
 STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const SRpcMsg *pReq);
 void    mndTransDrop(STrans *pTrans);
@@ -59,6 +61,7 @@ void    mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb);
 int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans);
 void    mndTransProcessRsp(SNodeMsg *pRsp);
 void    mndTransPullup(SMnode *pMnode);
+int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/inc/mndUser.h

@@ -24,9 +24,12 @@ extern "C" {
 
 int32_t   mndInitUser(SMnode *pMnode);
 void      mndCleanupUser(SMnode *pMnode);
-SUserObj *mndAcquireUser(SMnode *pMnode, char *userName);
+SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName);
 void      mndReleaseUser(SMnode *pMnode, SUserObj *pUser);
 
+// for trans test
+SSdbRaw *mndUserActionEncode(SUserObj *pUser);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/mnode/impl/src/mndUser.c

@@ -25,7 +25,6 @@
 #define USER_RESERVE_SIZE 64
 
 static int32_t  mndCreateDefaultUsers(SMnode *pMnode);
-static SSdbRaw *mndUserActionEncode(SUserObj *pUser);
 static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw);
 static int32_t  mndUserActionInsert(SSdb *pSdb, SUserObj *pUser);
 static int32_t  mndUserActionDelete(SSdb *pSdb, SUserObj *pUser);
@@ -90,7 +89,7 @@ static int32_t mndCreateDefaultUsers(SMnode *pMnode) {
   return 0;
 }
 
-static SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
+SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
   terrno = TSDB_CODE_OUT_OF_MEMORY;
 
   int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs);
@@ -238,7 +237,7 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) {
   return 0;
 }
 
-SUserObj *mndAcquireUser(SMnode *pMnode, char *userName) {
+SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName) {
   SSdb     *pSdb = pMnode->pSdb;
   SUserObj *pUser = sdbAcquire(pSdb, SDB_USER, userName);
   if (pUser == NULL) {
@@ -276,9 +275,6 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate
   }
   sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
 
-  char *param = strdup("====> test code to be deleted later <=====");
-  mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1);
-
   if (mndTransPrepare(pMnode, pTrans) != 0) {
     mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
     mndTransDrop(pTrans);

source/dnode/mnode/impl/src/mnode.c

@@ -43,9 +43,6 @@
 #include "mndUser.h"
 #include "mndVgroup.h"
 
-#define MQ_TIMER_MS    2000
-#define TRNAS_TIMER_MS 6000
-
 static void *mndBuildTimerMsg(int32_t *pContLen) {
   SMTimerReq timerReq = {0};
 
@@ -68,7 +65,7 @@ static void mndPullupTrans(void *param, void *tmrId) {
     tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
   }
 
-  taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer);
+  taosTmrReset(mndPullupTrans, tsTransPullupMs, pMnode, pMnode->timer, &pMnode->transTimer);
 }
 
 static void mndCalMqRebalance(void *param, void *tmrId) {
@@ -84,7 +81,7 @@ static void mndCalMqRebalance(void *param, void *tmrId) {
     tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg);
   }
 
-  taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer);
+  taosTmrReset(mndCalMqRebalance, tsMaRebalanceMs, pMnode, pMnode->timer, &pMnode->mqTimer);
 }
 
 static void mndPullupTelem(void *param, void *tmrId) {
@@ -106,12 +103,12 @@ static int32_t mndInitTimer(SMnode *pMnode) {
     return -1;
   }
 
-  if (taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer)) {
+  if (taosTmrReset(mndPullupTrans, tsTransPullupMs, pMnode, pMnode->timer, &pMnode->transTimer)) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
 
-  if (taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer)) {
+  if (taosTmrReset(mndCalMqRebalance, tsMaRebalanceMs, pMnode, pMnode->timer, &pMnode->mqTimer)) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }

source/dnode/mnode/impl/test/trans/CMakeLists.txt

-aux_source_directory(. MNODE_TRANS_TEST_SRC)
-add_executable(transTest ${MNODE_TRANS_TEST_SRC})
+add_executable(transTest1 "")
+target_sources(transTest1
+    PRIVATE
+    "${CMAKE_CURRENT_SOURCE_DIR}/trans1.cpp"
+)
 target_link_libraries(
-    transTest
+    transTest1
     PUBLIC sut
 )
+target_include_directories(
+    transTest1
+    PUBLIC "${TD_SOURCE_DIR}/include/dnode/mnode"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../../inc"
+)
+add_test(
+    NAME transTest1
+    COMMAND transTest1
+)
 
+add_executable(transTest2 "")
+target_sources(transTest2
+    PRIVATE
+    "${CMAKE_CURRENT_SOURCE_DIR}/trans2.cpp"
+)
+target_link_libraries(
+    transTest2
+    PUBLIC dnode mnode gtest_main
+)
+target_include_directories(
+    transTest2
+    PUBLIC "${TD_SOURCE_DIR}/include/dnode/mnode"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../../inc"
+)
 add_test(
-    NAME transTest
-    COMMAND transTest
+    NAME transTest2
+    COMMAND transTest2
 )

source/dnode/mnode/impl/test/trans/trans.cpp → source/dnode/mnode/impl/test/trans/trans1.cpp

@@ -11,10 +11,10 @@
 
 #include "sut.h"
 
-class MndTestTrans : public ::testing::Test {
+class MndTestTrans1 : public ::testing::Test {
  protected:
   static void SetUpTestSuite() {
-    test.Init("/tmp/mnode_test_trans", 9013);
+    test.Init("/tmp/mnode_test_trans1", 9013);
     const char* fqdn = "localhost";
     const char* firstEp = "localhost:9013";
     server2.Start("/tmp/mnode_test_trans2", fqdn, 9020, firstEp);
@@ -26,7 +26,7 @@ class MndTestTrans : public ::testing::Test {
   }
 
   static void KillThenRestartServer() {
-    char      file[PATH_MAX] = "/tmp/mnode_test_trans/mnode/data/sdb.data";
+    char      file[PATH_MAX] = "/tmp/mnode_test_trans1/mnode/data/sdb.data";
     TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
     int32_t   size = 3 * 1024 * 1024;
     void*     buffer = taosMemoryMalloc(size);
@@ -60,10 +60,10 @@ class MndTestTrans : public ::testing::Test {
   void TearDown() override {}
 };
 
-Testbase   MndTestTrans::test;
-TestServer MndTestTrans::server2;
+Testbase   MndTestTrans1::test;
+TestServer MndTestTrans1::server2;
 
-TEST_F(MndTestTrans, 00_Create_User_Crash) {
+TEST_F(MndTestTrans1, 00_Create_User_Crash) {
   {
     test.SendShowReq(TSDB_MGMT_TABLE_TRANS, "trans", "");
     EXPECT_EQ(test.GetShowRows(), 0);
@@ -83,7 +83,7 @@ TEST_F(MndTestTrans, 00_Create_User_Crash) {
   }
 }
 
-TEST_F(MndTestTrans, 01_Create_User_Crash) {
+TEST_F(MndTestTrans1, 01_Create_User_Crash) {
   {
     SCreateUserReq createReq = {0};
     strcpy(createReq.user, "u1");
@@ -107,7 +107,7 @@ TEST_F(MndTestTrans, 01_Create_User_Crash) {
   EXPECT_EQ(test.GetShowRows(), 2);
 }
 
-TEST_F(MndTestTrans, 02_Create_Qnode1_Crash) {
+TEST_F(MndTestTrans1, 02_Create_Qnode1_Crash) {
   {
     SMCreateQnodeReq createReq = {0};
     createReq.dnodeId = 1;
@@ -142,7 +142,7 @@ TEST_F(MndTestTrans, 02_Create_Qnode1_Crash) {
   }
 }
 
-TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) {
+TEST_F(MndTestTrans1, 03_Create_Qnode2_Crash) {
   {
     SCreateDnodeReq createReq = {0};
     strcpy(createReq.fqdn, "localhost");

source/dnode/mnode/impl/test/trans/trans2.cpp

+/**
+ * @file trans.cpp
+ * @author slguan (slguan@taosdata.com)
+ * @brief MNODE module trans tests
+ * @version 1.0
+ * @date 2022-05-02
+ *
+ * @copyright Copyright (c) 2022
+ *
+ */
+
+#include <gtest/gtest.h>
+
+#include "mndTrans.h"
+#include "mndUser.h"
+#include "tcache.h"
+
+void reportStartup(SMgmtWrapper *pWrapper, const char *name, const char *desc) {}
+
+class MndTestTrans2 : public ::testing::Test {
+ protected:
+  static void InitLog() {
+    dDebugFlag = 143;
+    vDebugFlag = 0;
+    mDebugFlag = 207;
+    cDebugFlag = 0;
+    jniDebugFlag = 0;
+    tmrDebugFlag = 135;
+    uDebugFlag = 135;
+    rpcDebugFlag = 143;
+    qDebugFlag = 0;
+    wDebugFlag = 0;
+    sDebugFlag = 0;
+    tsdbDebugFlag = 0;
+    tsLogEmbedded = 1;
+    tsAsyncLog = 0;
+
+    const char *logpath = "/tmp/td";
+    taosRemoveDir(logpath);
+    taosMkDir(logpath);
+    tstrncpy(tsLogDir, logpath, PATH_MAX);
+    if (taosInitLog("taosdlog", 1) != 0) {
+      printf("failed to init log file\n");
+    }
+  }
+
+  static void InitMnode() {
+    static SMsgCb msgCb = {0};
+    msgCb.reportStartupFp = reportStartup;
+    msgCb.pWrapper = (SMgmtWrapper *)(&msgCb);  // hack
+    tmsgSetDefaultMsgCb(&msgCb);
+
+    SMnodeOpt opt = {0};
+    opt.deploy = 1;
+    opt.replica = 1;
+    opt.replicas[0].id = 1;
+    opt.replicas[0].port = 9040;
+    strcpy(opt.replicas[0].fqdn, "localhost");
+    opt.msgCb = msgCb;
+
+    tsTransPullupMs = 1000;
+
+    const char *mnodepath = "/tmp/mnode_test_trans";
+    taosRemoveDir(mnodepath);
+    pMnode = mndOpen(mnodepath, &opt);
+    mndStart(pMnode);
+  }
+
+  static void SetUpTestSuite() {
+    InitLog();
+    walInit();
+    InitMnode();
+  }
+
+  static void TearDownTestSuite() {
+    mndStop(pMnode);
+    mndClose(pMnode);
+    walCleanUp();
+    taosCloseLog();
+    taosStopCacheRefreshWorker();
+  }
+
+  static SMnode *pMnode;
+
+ public:
+  void SetUp() override {}
+  void TearDown() override {}
+
+  int32_t CreateUserLog(const char *acct, const char *user, ETrnType type, SDbObj *pDb) {
+    SUserObj userObj = {0};
+    taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass);
+    tstrncpy(userObj.user, user, TSDB_USER_LEN);
+    tstrncpy(userObj.acct, acct, TSDB_USER_LEN);
+    userObj.createdTime = taosGetTimestampMs();
+    userObj.updateTime = userObj.createdTime;
+    userObj.superUser = 1;
+
+    SRpcMsg  rpcMsg = {0};
+    STrans  *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, type, &rpcMsg);
+    SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj);
+    mndTransAppendRedolog(pTrans, pRedoRaw);
+    sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
+
+    SSdbRaw *pUndoRaw = mndUserActionEncode(&userObj);
+    mndTransAppendUndolog(pTrans, pUndoRaw);
+    sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED);
+
+    char *param = strdup("====> test log <=====");
+    mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1);
+
+    if (pDb != NULL) {
+      mndTransSetDbInfo(pTrans, pDb);
+    }
+
+    int32_t code = mndTransPrepare(pMnode, pTrans);
+    mndTransDrop(pTrans);
+
+    return code;
+  }
+
+  int32_t CreateUserAction(const char *acct, const char *user, bool hasUndoAction, ETrnPolicy policy, ETrnType type,
+                           SDbObj *pDb) {
+    SUserObj userObj = {0};
+    taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass);
+    tstrncpy(userObj.user, user, TSDB_USER_LEN);
+    tstrncpy(userObj.acct, acct, TSDB_USER_LEN);
+    userObj.createdTime = taosGetTimestampMs();
+    userObj.updateTime = userObj.createdTime;
+    userObj.superUser = 1;
+
+    SRpcMsg  rpcMsg = {0};
+    STrans  *pTrans = mndTransCreate(pMnode, policy, type, &rpcMsg);
+    SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj);
+    mndTransAppendRedolog(pTrans, pRedoRaw);
+    sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
+
+    SSdbRaw *pUndoRaw = mndUserActionEncode(&userObj);
+    mndTransAppendUndolog(pTrans, pUndoRaw);
+    sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED);
+
+    char *param = strdup("====> test action <=====");
+    mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1);
+
+    {
+      STransAction action = {0};
+      action.epSet.inUse = 0;
+      action.epSet.numOfEps = 1;
+      action.epSet.eps[0].port = 9040;
+      strcpy(action.epSet.eps[0].fqdn, "localhost");
+
+      int32_t contLen = 1024;
+      void   *pReq = taosMemoryCalloc(1, contLen);
+      strcpy((char *)pReq, "hello world redo");
+      action.pCont = pReq;
+      action.contLen = contLen;
+      action.msgType = TDMT_DND_CREATE_MNODE;
+      action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED;
+      mndTransAppendRedoAction(pTrans, &action);
+    }
+
+    if (hasUndoAction) {
+      STransAction action = {0};
+      action.epSet.inUse = 0;
+      action.epSet.numOfEps = 1;
+      action.epSet.eps[0].port = 9040;
+      strcpy(action.epSet.eps[0].fqdn, "localhost");
+
+      int32_t contLen = 1024;
+      void   *pReq = taosMemoryCalloc(1, contLen);
+      strcpy((char *)pReq, "hello world undo");
+      action.pCont = pReq;
+      action.contLen = contLen;
+      action.msgType = TDMT_DND_CREATE_MNODE;
+      action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED;
+      mndTransAppendUndoAction(pTrans, &action);
+    }
+
+    {
+      void *pRsp = taosMemoryCalloc(1, 256);
+      strcpy((char *)pRsp, "simple rsponse");
+      mndTransSetRpcRsp(pTrans, pRsp, 256);
+    }
+
+    if (pDb != NULL) {
+      mndTransSetDbInfo(pTrans, pDb);
+    }
+
+    int32_t code = mndTransPrepare(pMnode, pTrans);
+    mndTransDrop(pTrans);
+
+    return code;
+  }
+
+  int32_t CreateUserGlobal(const char *acct, const char *user) {
+    SUserObj userObj = {0};
+    taosEncryptPass_c((uint8_t *)"taosdata", strlen("taosdata"), userObj.pass);
+    tstrncpy(userObj.user, user, TSDB_USER_LEN);
+    tstrncpy(userObj.acct, acct, TSDB_USER_LEN);
+    userObj.createdTime = taosGetTimestampMs();
+    userObj.updateTime = userObj.createdTime;
+    userObj.superUser = 1;
+
+    SRpcMsg  rpcMsg = {0};
+    STrans  *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, &rpcMsg);
+    SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj);
+    mndTransAppendRedolog(pTrans, pRedoRaw);
+    sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
+
+    SSdbRaw *pUndoRaw = mndUserActionEncode(&userObj);
+    mndTransAppendUndolog(pTrans, pUndoRaw);
+    sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED);
+
+    char *param = strdup("====> test log <=====");
+    mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1);
+
+    int32_t code = mndTransPrepare(pMnode, pTrans);
+    mndTransDrop(pTrans);
+
+    return code;
+  }
+};
+
+SMnode *MndTestTrans2::pMnode;
+
+TEST_F(MndTestTrans2, 01_Log) {
+  const char *acct = "root";
+  const char *acct_invalid = "root1";
+  const char *user1 = "log1";
+  const char *user2 = "log2";
+  SUserObj   *pUser1 = NULL;
+  SUserObj   *pUser2 = NULL;
+
+  ASSERT_NE(pMnode, nullptr);
+
+  EXPECT_EQ(CreateUserLog(acct, user1, TRN_TYPE_CREATE_USER, NULL), 0);
+  pUser1 = mndAcquireUser(pMnode, user1);
+  ASSERT_NE(pUser1, nullptr);
+
+  // failed to create user and rollback
+  EXPECT_EQ(CreateUserLog(acct_invalid, user2, TRN_TYPE_CREATE_USER, NULL), 0);
+  pUser2 = mndAcquireUser(pMnode, user2);
+  ASSERT_EQ(pUser2, nullptr);
+
+  mndTransPullup(pMnode);
+}
+
+TEST_F(MndTestTrans2, 02_Action) {
+  const char *acct = "root";
+  const char *acct_invalid = "root1";
+  const char *user1 = "action1";
+  const char *user2 = "action2";
+  SUserObj   *pUser1 = NULL;
+  SUserObj   *pUser2 = NULL;
+  STrans     *pTrans = NULL;
+  int32_t     transId = 0;
+  int32_t     action = 0;
+
+  ASSERT_NE(pMnode, nullptr);
+
+  {
+    // failed to create user and rollback
+    EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, NULL), 0);
+    pUser1 = mndAcquireUser(pMnode, user1);
+    ASSERT_EQ(pUser1, nullptr);
+    mndReleaseUser(pMnode, pUser1);
+
+    // create user, and fake a response
+    {
+      EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, NULL), 0);
+      pUser1 = mndAcquireUser(pMnode, user1);
+      ASSERT_NE(pUser1, nullptr);
+      mndReleaseUser(pMnode, pUser1);
+
+      transId = 4;
+      pTrans = mndAcquireTrans(pMnode, transId);
+      EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR);
+      EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION);
+      EXPECT_EQ(pTrans->failedTimes, 1);
+
+      STransAction *pAction = (STransAction *)taosArrayGet(pTrans->undoActions, action);
+      pAction->msgSent = 1;
+
+      SNodeMsg rspMsg = {0};
+      rspMsg.pNode = pMnode;
+      int64_t signature = transId;
+      signature = (signature << 32);
+      signature += action;
+      rspMsg.rpcMsg.ahandle = (void *)signature;
+      mndTransProcessRsp(&rspMsg);
+      mndReleaseTrans(pMnode, pTrans);
+
+      pUser1 = mndAcquireUser(pMnode, user1);
+      ASSERT_EQ(pUser1, nullptr);
+      mndReleaseUser(pMnode, pUser1);
+    }
+  }
+
+  {
+    EXPECT_EQ(CreateUserAction(acct, user1, false, TRN_POLICY_RETRY, TRN_TYPE_CREATE_USER, NULL), 0);
+    pUser1 = mndAcquireUser(pMnode, user1);
+    ASSERT_NE(pUser1, nullptr);
+    mndReleaseUser(pMnode, pUser1);
+
+    {
+      transId = 5;
+      pTrans = mndAcquireTrans(pMnode, transId);
+      EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR);
+      EXPECT_EQ(pTrans->stage, TRN_STAGE_REDO_ACTION);
+      EXPECT_EQ(pTrans->failedTimes, 1);
+
+      STransAction *pAction = (STransAction *)taosArrayGet(pTrans->redoActions, action);
+      pAction->msgSent = 1;
+
+      SNodeMsg rspMsg = {0};
+      rspMsg.pNode = pMnode;
+      int64_t signature = transId;
+      signature = (signature << 32);
+      signature += action;
+      rspMsg.rpcMsg.ahandle = (void *)signature;
+      rspMsg.rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
+      mndTransProcessRsp(&rspMsg);
+      mndReleaseTrans(pMnode, pTrans);
+
+      pUser1 = mndAcquireUser(pMnode, user1);
+      ASSERT_NE(pUser1, nullptr);
+      mndReleaseUser(pMnode, pUser1);
+    }
+
+    {
+      transId = 5;
+      pTrans = mndAcquireTrans(pMnode, transId);
+      EXPECT_EQ(pTrans->code, TSDB_CODE_RPC_NETWORK_UNAVAIL);
+      EXPECT_EQ(pTrans->stage, TRN_STAGE_REDO_ACTION);
+      EXPECT_EQ(pTrans->failedTimes, 2);
+
+      STransAction *pAction = (STransAction *)taosArrayGet(pTrans->redoActions, action);
+      pAction->msgSent = 1;
+
+      SNodeMsg rspMsg = {0};
+      rspMsg.pNode = pMnode;
+      int64_t signature = transId;
+      signature = (signature << 32);
+      signature += action;
+      rspMsg.rpcMsg.ahandle = (void *)signature;
+      mndTransProcessRsp(&rspMsg);
+      mndReleaseTrans(pMnode, pTrans);
+
+      pUser1 = mndAcquireUser(pMnode, user1);
+      ASSERT_NE(pUser1, nullptr);
+      mndReleaseUser(pMnode, pUser1);
+    }
+  }
+
+  {
+    EXPECT_EQ(CreateUserAction(acct, user2, true, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, NULL), 0);
+    SUserObj *pUser2 = (SUserObj *)sdbAcquire(pMnode->pSdb, SDB_USER, user2);
+    ASSERT_NE(pUser2, nullptr);
+    mndReleaseUser(pMnode, pUser2);
+
+    {
+      transId = 6;
+      pTrans = mndAcquireTrans(pMnode, transId);
+      EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR);
+      EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION);
+      EXPECT_EQ(pTrans->failedTimes, 1);
+
+      SNodeMsg rspMsg = {0};
+      rspMsg.pNode = pMnode;
+      int64_t signature = transId;
+      signature = (signature << 32);
+      signature += action;
+      rspMsg.rpcMsg.ahandle = (void *)signature;
+      rspMsg.rpcMsg.code = 0;
+      mndTransProcessRsp(&rspMsg);
+      mndReleaseTrans(pMnode, pTrans);
+
+      pUser2 = mndAcquireUser(pMnode, user2);
+      ASSERT_NE(pUser2, nullptr);
+      mndReleaseUser(pMnode, pUser2);
+    }
+
+    {
+      transId = 6;
+      pTrans = mndAcquireTrans(pMnode, transId);
+      EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR);
+      EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION);
+      EXPECT_EQ(pTrans->failedTimes, 2);
+
+      STransAction *pAction = (STransAction *)taosArrayGet(pTrans->undoActions, action);
+      pAction->msgSent = 1;
+
+      SNodeMsg rspMsg = {0};
+      rspMsg.pNode = pMnode;
+      int64_t signature = transId;
+      signature = (signature << 32);
+      signature += action;
+      rspMsg.rpcMsg.ahandle = (void *)signature;
+      mndTransProcessRsp(&rspMsg);
+      mndReleaseTrans(pMnode, pTrans);
+
+      pUser2 = mndAcquireUser(pMnode, user2);
+      ASSERT_EQ(pUser2, nullptr);
+      mndReleaseUser(pMnode, pUser2);
+    }
+  }
+}
+
+TEST_F(MndTestTrans2, 03_Kill) {
+  const char *acct = "root";
+  const char *user1 = "kill1";
+  const char *user2 = "kill2";
+  SUserObj   *pUser1 = NULL;
+  SUserObj   *pUser2 = NULL;
+  STrans     *pTrans = NULL;
+  int32_t     transId = 0;
+  int32_t     action = 0;
+
+  ASSERT_NE(pMnode, nullptr);
+
+  {
+    EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_USER, NULL), 0);
+    pUser1 = mndAcquireUser(pMnode, user1);
+    ASSERT_NE(pUser1, nullptr);
+    mndReleaseUser(pMnode, pUser1);
+
+    transId = 7;
+    pTrans = mndAcquireTrans(pMnode, transId);
+    EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR);
+    EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION);
+    EXPECT_EQ(pTrans->failedTimes, 1);
+
+    mndKillTrans(pMnode, pTrans);
+    mndReleaseTrans(pMnode, pTrans);
+
+    pUser1 = mndAcquireUser(pMnode, user1);
+    ASSERT_EQ(pUser1, nullptr);
+    mndReleaseUser(pMnode, pUser1);
+  }
+}
+
+TEST_F(MndTestTrans2, 04_Conflict) {
+  const char *acct = "root";
+  const char *user1 = "conflict1";
+  const char *user2 = "conflict2";
+  const char *user3 = "conflict3";
+  const char *user4 = "conflict4";
+  const char *user5 = "conflict5";
+  const char *user6 = "conflict6";
+  const char *user7 = "conflict7";
+  const char *user8 = "conflict8";
+  SUserObj   *pUser = NULL;
+  STrans     *pTrans = NULL;
+  int32_t     transId = 0;
+  int32_t     code = 0;
+
+  ASSERT_NE(pMnode, nullptr);
+
+  {
+    SDbObj dbObj1 = {0};
+    dbObj1.uid = 9521;
+    strcpy(dbObj1.name, "db");
+    SDbObj dbObj2 = {0};
+    dbObj2.uid = 9522;
+    strcpy(dbObj2.name, "conflict db");
+
+    EXPECT_EQ(CreateUserAction(acct, user1, true, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STB, &dbObj1), 0);
+    pUser = mndAcquireUser(pMnode, user1);
+    ASSERT_NE(pUser, nullptr);
+    mndReleaseUser(pMnode, pUser);
+
+    transId = 8;
+    pTrans = mndAcquireTrans(pMnode, transId);
+    EXPECT_EQ(pTrans->code, TSDB_CODE_INVALID_PTR);
+    EXPECT_EQ(pTrans->stage, TRN_STAGE_UNDO_ACTION);
+
+    // stb scope
+    EXPECT_EQ(CreateUserLog(acct, user2, TRN_TYPE_CREATE_DNODE, NULL), -1);
+    code = terrno;
+    EXPECT_EQ(code, TSDB_CODE_MND_TRANS_CONFLICT);
+
+    EXPECT_EQ(CreateUserLog(acct, user2, TRN_TYPE_CREATE_DB, &dbObj1), -1);
+    EXPECT_EQ(CreateUserLog(acct, user2, TRN_TYPE_CREATE_DB, &dbObj2), 0);
+    EXPECT_EQ(CreateUserLog(acct, user3, TRN_TYPE_CREATE_STB, &dbObj1), 0);
+
+    // db scope
+    pTrans->type = TRN_TYPE_CREATE_DB;
+    EXPECT_EQ(CreateUserLog(acct, user4, TRN_TYPE_CREATE_DNODE, NULL), -1);
+    EXPECT_EQ(CreateUserLog(acct, user4, TRN_TYPE_CREATE_DB, &dbObj1), -1);
+    EXPECT_EQ(CreateUserLog(acct, user4, TRN_TYPE_CREATE_DB, &dbObj2), 0);
+    EXPECT_EQ(CreateUserLog(acct, user5, TRN_TYPE_CREATE_STB, &dbObj1), -1);
+    EXPECT_EQ(CreateUserLog(acct, user5, TRN_TYPE_CREATE_STB, &dbObj2), 0);
+
+    // global scope
+    pTrans->type = TRN_TYPE_CREATE_DNODE;
+    EXPECT_EQ(CreateUserLog(acct, user6, TRN_TYPE_CREATE_DNODE, NULL), 0);
+    EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_DB, &dbObj1), -1);
+    EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_DB, &dbObj2), -1);
+    EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_STB, &dbObj1), -1);
+    EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_STB, &dbObj2), -1);
+
+    // global scope
+    pTrans->type = TRN_TYPE_CREATE_USER;
+    EXPECT_EQ(CreateUserLog(acct, user7, TRN_TYPE_CREATE_DB, &dbObj1), 0);
+    EXPECT_EQ(CreateUserLog(acct, user8, TRN_TYPE_CREATE_DB, &dbObj2), 0);
+
+    mndKillTrans(pMnode, pTrans);
+    mndReleaseTrans(pMnode, pTrans);
+
+    pUser = mndAcquireUser(pMnode, user1);
+    ASSERT_EQ(pUser, nullptr);
+    mndReleaseUser(pMnode, pUser);
+  }
+}
\ No newline at end of file

source/dnode/vnode/inc/vnode.h

@@ -37,7 +37,7 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-#define TSDB_VNODE_SMA_DEBUG  // TODO: evaluate to remove the macro and the relative codes
+
 // vnode
 typedef struct SVnode    SVnode;
 typedef struct STsdbCfg  STsdbCfg;  // todo: remove
@@ -145,7 +145,7 @@ struct STsdbCfg {
   int32_t keep2;
   // TODO: save to tsdb cfg file
   int8_t     type;  // ETsdbType
-  SRetention retentions[TSDB_RSMA_RETENTION_MAX];
+  SRetention retentions[TSDB_RETENTION_MAX];
 };
 
 typedef enum {

source/dnode/vnode/src/inc/meta.h

@@ -103,8 +103,6 @@ typedef struct {
 
 #if 1
 
-// int             metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
-int          metaDropTable(SMeta* pMeta, tb_uid_t uid);
 SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid);
 void         metaCloseSmaCursor(SMSmaCursor* pSmaCur);
 int64_t      metaSmaCursorNext(SMSmaCursor* pSmaCur);

source/dnode/vnode/src/inc/tsdb.h

@@ -72,6 +72,7 @@ struct STsdb {
   char          *path;
   SVnode        *pVnode;
   bool           repoLocked;
+  int8_t         level;  // retention level
   TdThreadMutex  mutex;
   STsdbMemTable *mem;
   STsdbMemTable *imem;
@@ -185,6 +186,7 @@ struct STsdbFS {
 
 #define REPO_ID(r)        TD_VID((r)->pVnode)
 #define REPO_CFG(r)       (&(r)->pVnode->config.tsdbCfg)
+#define REPO_LEVEL(r)     ((r)->level)
 #define REPO_FS(r)        ((r)->fs)
 #define REPO_META(r)      ((r)->pVnode->pMeta)
 #define REPO_TFS(r)       ((r)->pVnode->pTfs)

source/dnode/vnode/src/inc/vnodeInt.h

@@ -46,13 +46,13 @@
 extern "C" {
 #endif
 
-typedef struct SVnodeInfo   SVnodeInfo;
-typedef struct SMeta        SMeta;
-typedef struct STsdb        STsdb;
-typedef struct STQ          STQ;
-typedef struct SVState      SVState;
-typedef struct SVBufPool    SVBufPool;
-typedef struct SQWorker SQHandle;
+typedef struct SVnodeInfo SVnodeInfo;
+typedef struct SMeta      SMeta;
+typedef struct STsdb      STsdb;
+typedef struct STQ        STQ;
+typedef struct SVState    SVState;
+typedef struct SVBufPool  SVBufPool;
+typedef struct SQWorker   SQHandle;
 
 #define VNODE_META_DIR  "meta"
 #define VNODE_TSDB_DIR  "tsdb"
@@ -77,6 +77,7 @@ int             metaCommit(SMeta* pMeta);
 int             metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
 int             metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq);
 int             metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq);
+int             metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq);
 SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline);
 STSchema*       metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver);
 int             metaGetTableEntryByName(SMetaReader* pReader, const char* name);
@@ -100,7 +101,7 @@ int32_t      tsdbCreateTSma(STsdb* pTsdb, char* pMsg);
 int32_t      tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg);
 int          tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp);
 tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId,
-                              uint64_t taskId);
+                             uint64_t taskId);
 tsdbReaderT  tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId,
                                  void* pMemRef);
 int32_t      tsdbGetTableGroupFromIdListT(STsdb* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo);
@@ -124,6 +125,7 @@ int32_t tsdbUpdateTbUidList(STsdb* pTsdb, STbUidStore* pUidStore);
 void    tsdbUidStoreDestory(STbUidStore* pStore);
 void*   tsdbUidStoreFree(STbUidStore* pStore);
 int32_t tsdbTriggerRSma(STsdb* pTsdb, void* pMsg, int32_t inputType);
+int32_t tsdbProcessSubmitReq(STsdb* pTsdb, int64_t version, void* pReq);
 
 typedef struct {
   int8_t  streamType;  // sma or other
@@ -181,13 +183,13 @@ struct SVnode {
 
 struct STbUidStore {
   tb_uid_t  suid;
+  tb_uid_t  uid;  // TODO: just for debugging, remove when uid provided in SSDataBlock
   SArray*   tbUids;
   SHashObj* uidHash;
 };
 
 #define TD_VID(PVNODE) (PVNODE)->config.vgId
 
-
 static FORCE_INLINE bool vnodeIsRollup(SVnode* pVnode) {
   SRetention* pRetention = &(pVnode->config.tsdbCfg.retentions[0]);
   return (pRetention->freq > 0 && pRetention->keep > 0);

source/dnode/vnode/src/meta/metaTDBImpl.c

@@ -289,7 +289,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) {
   pVal = pBuf = buf;
   metaEncodeTbInfo(&pBuf, pTbCfg);
   vLen = POINTER_DISTANCE(pBuf, buf);
-  ret = tdbDbPut(pMetaDb->pTbDB, pKey, kLen, pVal, vLen, &pMetaDb->txn);
+  ret = tdbDbInsert(pMetaDb->pTbDB, pKey, kLen, pVal, vLen, &pMetaDb->txn);
   if (ret < 0) {
     return -1;
   }
@@ -311,7 +311,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) {
     pVal = pBuf = buf;
     metaEncodeSchemaEx(&pBuf, &schemaWrapper);
     vLen = POINTER_DISTANCE(pBuf, buf);
-    ret = tdbDbPut(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen, &pMeta->pDB->txn);
+    ret = tdbDbInsert(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen, &pMeta->pDB->txn);
     if (ret < 0) {
       return -1;
     }
@@ -325,7 +325,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) {
   kLen = nameLen + 1 + sizeof(uid);
   pVal = NULL;
   vLen = 0;
-  ret = tdbDbPut(pMetaDb->pNameIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
+  ret = tdbDbInsert(pMetaDb->pNameIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
   if (ret < 0) {
     return -1;
   }
@@ -336,7 +336,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) {
     kLen = sizeof(uid);
     pVal = NULL;
     vLen = 0;
-    ret = tdbDbPut(pMetaDb->pStbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
+    ret = tdbDbInsert(pMetaDb->pStbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
     if (ret < 0) {
       return -1;
     }
@@ -347,7 +347,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) {
     kLen = sizeof(ctbIdxKey);
     pVal = NULL;
     vLen = 0;
-    ret = tdbDbPut(pMetaDb->pCtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
+    ret = tdbDbInsert(pMetaDb->pCtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
     if (ret < 0) {
       return -1;
     }
@@ -362,7 +362,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) {
     kLen = sizeof(uid);
     pVal = NULL;
     vLen = 0;
-    ret = tdbDbPut(pMetaDb->pNtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
+    ret = tdbDbInsert(pMetaDb->pNtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
     if (ret < 0) {
       return -1;
     }
@@ -530,7 +530,7 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) {
   int32_t kLen = sizeof(pSmaCfg->indexUid);
   int32_t vLen = POINTER_DISTANCE(qBuf, pBuf);
 
-  ret = tdbDbPut(pMeta->pDB->pSmaDB, key, kLen, val, vLen, &pMetaDb->txn);
+  ret = tdbDbInsert(pMeta->pDB->pSmaDB, key, kLen, val, vLen, &pMetaDb->txn);
   if (ret < 0) {
     taosMemoryFreeClear(pBuf);
     return -1;
@@ -545,7 +545,7 @@ int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) {
   val = NULL;
   vLen = 0;
 
-  ret = tdbDbPut(pMeta->pDB->pSmaIdx, key, kLen, val, vLen, &pMetaDb->txn);
+  ret = tdbDbInsert(pMeta->pDB->pSmaIdx, key, kLen, val, vLen, &pMetaDb->txn);
   if (ret < 0) {
     taosMemoryFreeClear(pBuf);
     return -1;

source/dnode/vnode/src/meta/metaTable.c

@@ -72,44 +72,61 @@ _err:
 }
 
 int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) {
-  SMetaReader mr = {0};
-
-  // validate req
-  metaReaderInit(&mr, pMeta, 0);
-  if (metaGetTableEntryByUid(&mr, pReq->suid) < 0) {
-    terrno = TSDB_CODE_VND_TABLE_NOT_EXIST;
+  TDBC       *pNameIdxc = NULL;
+  TDBC       *pUidIdxc = NULL;
+  TDBC       *pCtbIdxc = NULL;
+  SCtbIdxKey *pCtbIdxKey;
+  const void *pKey = NULL;
+  int         nKey;
+  const void *pData = NULL;
+  int         nData;
+  int         c, ret;
+
+  // prepare uid idx cursor
+  tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
+  ret = tdbDbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c);
+  if (ret < 0 || c != 0) {
+    terrno = TSDB_CODE_VND_TB_NOT_EXIST;
+    tdbDbcClose(pUidIdxc);
     goto _err;
   }
 
-  // do drop
-  // drop from pTbDb
-  // drop from pSkmDb
-  // drop from pUidIdx
-  // drop from pNameIdx
-  // {
-  //   TDBC *pDbc1 = NULL;
-  //   void *pKey = NULL;
-  //   void *pVal = NULL;
-  //   int   kLen = 0;
-  //   int   vLen = 0;
-  //   int   ret = 0;
-
-  //   // drop from pCtbIdx
-  //   ret = tdbDbcOpen(pMeta->pCtbIdx, &pDbc1);
-  //   tdbDbcMoveTo(pDbc1, &pReq->suid, sizeof(pReq->suid), NULL /*cmpr*/, 0 /*TDB_FORWARD_SEARCH*/);
-  //   tdbDbcGet(pDbc1, &pKey, &kLen, &pVal, vLen);
-  //   tdbDbcDrop(pDbc1);
-  //   // drop from pTagIdx
-  //   // drop from pTtlIdx
-  // }
-
-  // clear and return
-  metaReaderClear(&mr);
-  metaError("vgId:%d  super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid);
+  // prepare name idx cursor
+  tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn);
+  ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c);
+  if (ret < 0 || c != 0) {
+    ASSERT(0);
+  }
+
+  tdbDbcDelete(pUidIdxc);
+  tdbDbcDelete(pNameIdxc);
+  tdbDbcClose(pUidIdxc);
+  tdbDbcClose(pNameIdxc);
+
+  // loop to drop each child table
+  tdbDbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn);
+  ret = tdbDbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c);
+  if (ret < 0 || (c < 0 && tdbDbcMoveToNext(pCtbIdxc) < 0)) {
+    tdbDbcClose(pCtbIdxc);
+    goto _exit;
+  }
+
+  for (;;) {
+    tdbDbcGet(pCtbIdxc, &pKey, &nKey, NULL, NULL);
+    pCtbIdxKey = (SCtbIdxKey *)pKey;
+
+    if (pCtbIdxKey->suid > pReq->suid) break;
+
+    // drop the child table (TODO)
+
+    if (tdbDbcMoveToNext(pCtbIdxc) < 0) break;
+  }
+
+_exit:
+  metaDebug("vgId:%d  super table %s uid:%" PRId64 " is dropped", TD_VID(pMeta->pVnode), pReq->name, pReq->suid);
   return 0;
 
 _err:
-  metaReaderClear(&mr);
   metaError("vgId:%d failed to drop super table %s uid:%" PRId64 " since %s", TD_VID(pMeta->pVnode), pReq->name,
             pReq->suid, tstrerror(terrno));
   return -1;
@@ -166,18 +183,122 @@ _err:
   return -1;
 }
 
-int metaDropTable(SMeta *pMeta, tb_uid_t uid) {
-#if 0
-  if (metaRemoveTableFromIdx(pMeta, uid) < 0) {
-    // TODO: handle error
+int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq) {
+  TDBC       *pTbDbc = NULL;
+  TDBC       *pUidIdxc = NULL;
+  TDBC       *pNameIdxc = NULL;
+  const void *pData;
+  int         nData;
+  tb_uid_t    uid;
+  int64_t     tver;
+  SMetaEntry  me = {0};
+  SCoder      coder = {0};
+  int8_t      type;
+  int64_t     ctime;
+  tb_uid_t    suid;
+  int         c, ret;
+
+  // search & delete the name idx
+  tdbDbcOpen(pMeta->pNameIdx, &pNameIdxc, &pMeta->txn);
+  ret = tdbDbcMoveTo(pNameIdxc, pReq->name, strlen(pReq->name) + 1, &c);
+  if (ret < 0 || c) {
+    tdbDbcClose(pNameIdxc);
+    terrno = TSDB_CODE_VND_TABLE_NOT_EXIST;
     return -1;
   }
 
-  if (metaRemoveTableFromIdx(pMeta, uid) < 0) {
-    // TODO
+  ret = tdbDbcGet(pNameIdxc, NULL, NULL, &pData, &nData);
+  if (ret < 0) {
+    ASSERT(0);
     return -1;
   }
-#endif
+
+  uid = *(tb_uid_t *)pData;
+
+  tdbDbcDelete(pNameIdxc);
+  tdbDbcClose(pNameIdxc);
+
+  // search & delete uid idx
+  tdbDbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
+  ret = tdbDbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c);
+  if (ret < 0 || c != 0) {
+    ASSERT(0);
+    return -1;
+  }
+
+  ret = tdbDbcGet(pUidIdxc, NULL, NULL, &pData, &nData);
+  if (ret < 0) {
+    ASSERT(0);
+    return -1;
+  }
+
+  tver = *(int64_t *)pData;
+  tdbDbcDelete(pUidIdxc);
+  tdbDbcClose(pUidIdxc);
+
+  // search and get meta entry
+  tdbDbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn);
+  ret = tdbDbcMoveTo(pTbDbc, &(STbDbKey){.uid = uid, .version = tver}, sizeof(STbDbKey), &c);
+  if (ret < 0 || c != 0) {
+    ASSERT(0);
+    return -1;
+  }
+
+  ret = tdbDbcGet(pTbDbc, NULL, NULL, &pData, &nData);
+  if (ret < 0) {
+    ASSERT(0);
+    return -1;
+  }
+
+  // decode entry
+  void *pDataCopy = taosMemoryMalloc(nData);  // remove the copy (todo)
+  memcpy(pDataCopy, pData, nData);
+  tCoderInit(&coder, TD_LITTLE_ENDIAN, pDataCopy, nData, TD_DECODER);
+  ret = metaDecodeEntry(&coder, &me);
+  if (ret < 0) {
+    ASSERT(0);
+    return -1;
+  }
+
+  type = me.type;
+  if (type == TSDB_CHILD_TABLE) {
+    ctime = me.ctbEntry.ctime;
+    suid = me.ctbEntry.suid;
+  } else if (type == TSDB_NORMAL_TABLE) {
+    ctime = me.ntbEntry.ctime;
+    suid = 0;
+  } else {
+    ASSERT(0);
+  }
+
+  taosMemoryFree(pDataCopy);
+  tCoderClear(&coder);
+  tdbDbcClose(pTbDbc);
+
+  if (type == TSDB_CHILD_TABLE) {
+    // remove the pCtbIdx
+    TDBC *pCtbIdxc = NULL;
+    tdbDbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn);
+
+    ret = tdbDbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = uid}, sizeof(SCtbIdxKey), &c);
+    if (ret < 0 || c != 0) {
+      ASSERT(0);
+      return -1;
+    }
+
+    tdbDbcDelete(pCtbIdxc);
+    tdbDbcClose(pCtbIdxc);
+
+    // remove tags from pTagIdx (todo)
+  } else if (type == TSDB_NORMAL_TABLE) {
+    // remove from pSkmDb
+  } else {
+    ASSERT(0);
+  }
+
+  // remove from ttl (todo)
+  if (ctime > 0) {
+  }
 
   return 0;
 }
@@ -218,7 +339,7 @@ static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) {
   tCoderClear(&coder);
 
   // write to table.db
-  if (tdbDbPut(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) {
+  if (tdbDbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) {
     goto _err;
   }
 
@@ -231,11 +352,11 @@ _err:
 }
 
 static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) {
-  return tdbDbPut(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn);
+  return tdbDbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn);
 }
 
 static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) {
-  return tdbDbPut(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn);
+  return tdbDbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn);
 }
 
 static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
@@ -258,12 +379,12 @@ static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
   ttlKey.dtime = ctime + ttlDays * 24 * 60 * 60;
   ttlKey.uid = pME->uid;
 
-  return tdbDbPut(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn);
+  return tdbDbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn);
 }
 
 static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) {
   SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid};
-  return tdbDbPut(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn);
+  return tdbDbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn);
 }
 
 static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pME) {
@@ -304,7 +425,7 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) {
   tCoderInit(&coder, TD_LITTLE_ENDIAN, pVal, vLen, TD_ENCODER);
   tEncodeSSchemaWrapper(&coder, pSW);
 
-  if (tdbDbPut(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) {
+  if (tdbDbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) {
     rcode = -1;
     goto _exit;
   }

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -15,6 +15,8 @@
 
 #include "tsdb.h"
 
+extern const char *TSDB_LEVEL_DNAME[];
+
 typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T;
 static const char *tsdbTxnFname[] = {"current.t", "current"};
 #define TSDB_MAX_FSETS(keep, days) ((keep) / (days) + 3)
@@ -35,12 +37,12 @@ static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired);
 // static int  tsdbProcessExpiredFS(STsdb *pRepo);
 // static int  tsdbCreateMeta(STsdb *pRepo);
 
-static void tsdbGetRootDir(int repoid, char dirName[]) {
-  snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb", repoid);
+static void tsdbGetRootDir(int repoid, int8_t level, char dirName[]) {
+  snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s", repoid, TSDB_LEVEL_DNAME[level]);
 }
 
-static void tsdbGetDataDir(int repoid, char dirName[]) {
-  snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data", repoid);
+static void tsdbGetDataDir(int repoid, int8_t level, char dirName[]) {
+  snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data", repoid, TSDB_LEVEL_DNAME[level]);
 }
 
 // For backward compatibility
@@ -588,8 +590,8 @@ static int tsdbComparFidFSet(const void *arg1, const void *arg2) {
 }
 
 static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) {
-  snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), REPO_ID(pRepo),
-           tsdbTxnFname[ftype]);
+  snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/%s/%s", tfsGetPrimaryPath(REPO_TFS(pRepo)), REPO_ID(pRepo),
+           TSDB_LEVEL_DNAME[REPO_LEVEL(pRepo)], tsdbTxnFname[ftype]);
 }
 
 static int tsdbOpenFSFromCurrent(STsdb *pRepo) {
@@ -719,7 +721,7 @@ static int tsdbScanRootDir(STsdb *pRepo) {
   STsdbFS        *pfs = REPO_FS(pRepo);
   const STfsFile *pf;
 
-  tsdbGetRootDir(REPO_ID(pRepo), rootDir);
+  tsdbGetRootDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), rootDir);
   STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), rootDir);
   if (tdir == NULL) {
     tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno));
@@ -753,7 +755,7 @@ static int tsdbScanDataDir(STsdb *pRepo) {
   STsdbFS        *pfs = REPO_FS(pRepo);
   const STfsFile *pf;
 
-  tsdbGetDataDir(REPO_ID(pRepo), dataDir);
+  tsdbGetDataDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), dataDir);
   STfsDir *tdir = tfsOpendir(REPO_TFS(pRepo), dataDir);
   if (tdir == NULL) {
     tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno));
@@ -801,7 +803,7 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) {
   regex_t         regex;
   STsdbFS        *pfs = REPO_FS(pRepo);
 
-  tsdbGetDataDir(REPO_ID(pRepo), dataDir);
+  tsdbGetDataDir(REPO_ID(pRepo), REPO_LEVEL(pRepo), dataDir);
 
   // Resource allocation and init
   regcomp(&regex, pattern, REG_EXTENDED);

source/dnode/vnode/src/tsdb/tsdbFile.c

@@ -27,7 +27,13 @@ static const char *TSDB_FNAME_SUFFIX[] = {
     "rsma",  // TSDB_FILE_RSMA
 };
 
-static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname);
+const char *TSDB_LEVEL_DNAME[] = {
+    "tsdb",
+    "rsma1",
+    "rsma2",
+};
+
+static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char* dname, char *fname);
 // static int   tsdbRollBackMFile(SMFile *pMFile);
 static int   tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo);
 static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo);
@@ -45,7 +51,7 @@ void tsdbInitDFile(STsdb *pRepo, SDFile *pDFile, SDiskID did, int fid, uint32_t
   pDFile->info.magic = TSDB_FILE_INIT_MAGIC;
   pDFile->info.fver = tsdbGetDFSVersion(ftype);
 
-  tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, fname);
+  tsdbGetFilename(REPO_ID(pRepo), fid, ver, ftype, TSDB_LEVEL_DNAME[pRepo->level], fname);
   tfsInitFile(REPO_TFS(pRepo), &(pDFile->f), did, fname);
 }
 
@@ -431,14 +437,15 @@ int tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype
   return 0;
 }
 
-static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname) {
+static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, const char *dname, char *fname) {
   ASSERT(ftype != TSDB_FILE_MAX);
 
   if (ftype < TSDB_FILE_MAX) {
     if (ver == 0) {
-      snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data/v%df%d.%s", vid, vid, fid, TSDB_FNAME_SUFFIX[ftype]);
+      snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data/v%df%d.%s", vid, dname, vid, fid,
+               TSDB_FNAME_SUFFIX[ftype]);
     } else {
-      snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data/v%df%d.%s-ver%" PRIu32, vid, vid, fid,
+      snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/data/v%df%d.%s-ver%" PRIu32, vid, dname, vid, fid,
                TSDB_FNAME_SUFFIX[ftype], ver);
     }
   } else {

source/dnode/vnode/src/tsdb/tsdbOpen.c

@@ -15,21 +15,21 @@
 
 #include "tsdb.h"
 
-static int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir);
+static int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir, int8_t level);
 
 int tsdbOpen(SVnode *pVnode, int8_t type) {
   switch (type) {
     case TSDB_TYPE_TSDB:
-      return tsdbOpenImpl(pVnode, type, &VND_TSDB(pVnode), VNODE_TSDB_DIR);
+      return tsdbOpenImpl(pVnode, type, &VND_TSDB(pVnode), VNODE_TSDB_DIR, TSDB_RETENTION_L0);
     case TSDB_TYPE_TSMA:
       ASSERT(0);
       break;
     case TSDB_TYPE_RSMA_L0:
-      return tsdbOpenImpl(pVnode, type, &VND_RSMA0(pVnode), VNODE_TSDB_DIR);
+      return tsdbOpenImpl(pVnode, type, &VND_RSMA0(pVnode), VNODE_TSDB_DIR, TSDB_RETENTION_L0);
     case TSDB_TYPE_RSMA_L1:
-      return tsdbOpenImpl(pVnode, type, &VND_RSMA1(pVnode), VNODE_RSMA1_DIR);
+      return tsdbOpenImpl(pVnode, type, &VND_RSMA1(pVnode), VNODE_RSMA1_DIR, TSDB_RETENTION_L1);
     case TSDB_TYPE_RSMA_L2:
-      return tsdbOpenImpl(pVnode, type, &VND_RSMA2(pVnode), VNODE_RSMA2_DIR);
+      return tsdbOpenImpl(pVnode, type, &VND_RSMA2(pVnode), VNODE_RSMA2_DIR, TSDB_RETENTION_L2);
     default:
       ASSERT(0);
       break;
@@ -37,7 +37,17 @@ int tsdbOpen(SVnode *pVnode, int8_t type) {
   return 0;
 }
 
-int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir) {
+/**
+ * @brief 
+ * 
+ * @param pVnode 
+ * @param type 
+ * @param ppTsdb 
+ * @param dir 
+ * @param level retention level
+ * @return int 
+ */
+int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir, int8_t level) {
   STsdb *pTsdb = NULL;
   int    slen = 0;
 
@@ -55,6 +65,7 @@ int tsdbOpenImpl(SVnode *pVnode, int8_t type, STsdb **ppTsdb, const char *dir) {
   sprintf(pTsdb->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP,
           dir);
   pTsdb->pVnode = pVnode;
+  pTsdb->level = level;
   pTsdb->repoLocked = false;
   taosThreadMutexInit(&pTsdb->mutex, NULL);
   pTsdb->fs = tsdbNewFS(REPO_CFG(pTsdb));
@@ -79,6 +90,7 @@ _err:
 
 int tsdbClose(STsdb *pTsdb) {
   if (pTsdb) {
+    // TODO: destroy mem/imem
     tsdbCloseFS(pTsdb);
     tsdbFreeFS(pTsdb->fs);
     taosMemoryFree(pTsdb);

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -98,46 +98,46 @@ typedef struct SIOCostSummary {
 } SIOCostSummary;
 
 typedef struct SBlockLoadSuppInfo {
-  SColumnDataAgg  *pstatis;
-  SColumnDataAgg **plist;
-  SArray          *defaultLoadColumn;  // default load column
-  int32_t         *slotIds;            // colId to slotId
+  SColumnDataAgg*  pstatis;
+  SColumnDataAgg** plist;
+  SArray*          defaultLoadColumn;  // default load column
+  int32_t*         slotIds;            // colId to slotId
 } SBlockLoadSuppInfo;
 
 typedef struct STsdbReadHandle {
-  STsdb*          pTsdb;
-  SQueryFilePos   cur;  // current position
-  int16_t         order;
-  STimeWindow     window;  // the primary query time window that applies to all queries
-//  SColumnDataAgg* statis;  // query level statistics, only one table block statistics info exists at any time
-//  SColumnDataAgg** pstatis;// the ptr array list to return to caller
-  int32_t         numOfBlocks;
-  SArray*         pColumns;  // column list, SColumnInfoData array list
-  bool            locateStart;
-  int32_t         outputCapacity;
-  int32_t         realNumOfRows;
-  SArray*         pTableCheckInfo;  // SArray<STableCheckInfo>
-  int32_t         activeIndex;
-  bool            checkFiles;               // check file stage
-  int8_t          cachelastrow;             // check if last row cached
-  bool            loadExternalRow;          // load time window external data rows
-  bool            currentLoadExternalRows;  // current load external rows
-  int32_t         loadType;                 // block load type
-  char*           idStr;                    // query info handle, for debug purpose
+  STsdb*        pTsdb;
+  SQueryFilePos cur;  // current position
+  int16_t       order;
+  STimeWindow   window;  // the primary query time window that applies to all queries
+  //  SColumnDataAgg* statis;  // query level statistics, only one table block statistics info exists at any time
+  //  SColumnDataAgg** pstatis;// the ptr array list to return to caller
+  int32_t numOfBlocks;
+  SArray* pColumns;  // column list, SColumnInfoData array list
+  bool    locateStart;
+  int32_t outputCapacity;
+  int32_t realNumOfRows;
+  SArray* pTableCheckInfo;  // SArray<STableCheckInfo>
+  int32_t activeIndex;
+  bool    checkFiles;               // check file stage
+  int8_t  cachelastrow;             // check if last row cached
+  bool    loadExternalRow;          // load time window external data rows
+  bool    currentLoadExternalRows;  // current load external rows
+  int32_t loadType;                 // block load type
+  char*   idStr;                    // query info handle, for debug purpose
   int32_t type;  // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows
   SDFileSet*         pFileGroup;
   SFSIter            fileIter;
   SReadH             rhelper;
   STableBlockInfo*   pDataBlockInfo;
-  SDataCols*         pDataCols;          // in order to hold current file data block
-  int32_t            allocSize;          // allocated data block size
-  SDataBlockLoadInfo dataBlockLoadInfo;  /* record current block load information */
-  SLoadCompBlockInfo compBlockLoadInfo;  /* record current compblock information in SQueryAttr */
+  SDataCols*         pDataCols;         // in order to hold current file data block
+  int32_t            allocSize;         // allocated data block size
+  SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */
+  SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQueryAttr */
   SBlockLoadSuppInfo suppInfo;
-  SArray*        prev;  // previous row which is before than time window
-  SArray*        next;  // next row which is after the query time window
-  SIOCostSummary cost;
-  STSchema*      pSchema;
+  SArray*            prev;  // previous row which is before than time window
+  SArray*            next;  // next row which is after the query time window
+  SIOCostSummary     cost;
+  STSchema*          pSchema;
 } STsdbReadHandle;
 
 typedef struct STableGroupSupporter {
@@ -163,6 +163,8 @@ static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2);
 // static void*   destroyTableCheckInfo(SArray* pTableCheckInfo);
 static bool tsdbGetExternalRow(tsdbReaderT pHandle);
 
+static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions);
+
 static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) {
   pBlockLoadInfo->slot = -1;
   pBlockLoadInfo->uid = 0;
@@ -349,12 +351,38 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableData
               pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr);
   }
 }
-
+#if 0
+int           nQUERY = 0;
+#endif
 static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions) {
   if (vnodeIsRollup(pVnode)) {
-    // for(int32_t i=0; i< TSDB_; ) {
-      
-    // }
+    int level = 0;
+#if 1
+    int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision);
+    for (int i = 0; i < TSDB_RETENTION_MAX; ++i) {
+      SRetention* pRetention = retentions + i;
+      if (pRetention->keep <= 0 || (now - pRetention->keep) >= winSKey) {
+        break;
+      }
+    }
+#endif
+#if 0
+    ++nQUERY;
+    if(nQUERY%3 == 0) {
+      level = 2;
+    } else if(nQUERY%2 == 0) {
+      level = 1;
+    } else {
+      level = 0;
+    }
+#endif
+    if (level == TSDB_RETENTION_L0) {
+      return VND_RSMA0(pVnode);
+    } else if (level == TSDB_RETENTION_L1) {
+      return VND_RSMA1(pVnode);
+    } else {
+      return VND_RSMA2(pVnode);
+    }
   }
   return pVnode->pTsdb;
 }
@@ -419,8 +447,10 @@ static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond*
     }
 
     pReadHandle->suppInfo.defaultLoadColumn = getDefaultLoadColumns(pReadHandle, true);
-    pReadHandle->suppInfo.slotIds = taosMemoryMalloc(sizeof(int32_t) * taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn));
-    pReadHandle->suppInfo.plist = taosMemoryCalloc(taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn), POINTER_BYTES);
+    pReadHandle->suppInfo.slotIds =
+        taosMemoryMalloc(sizeof(int32_t) * taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn));
+    pReadHandle->suppInfo.plist =
+        taosMemoryCalloc(taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn), POINTER_BYTES);
   }
 
   pReadHandle->pDataCols = tdNewDataCols(1000, pVnode->config.tsdbCfg.maxRows);
@@ -443,7 +473,6 @@ _end:
 
 tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId,
                              uint64_t taskId) {
-
   STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(pVnode, pCond, qId, taskId);
   if (pTsdbReadHandle == NULL) {
     return NULL;
@@ -461,16 +490,16 @@ tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableG
     return NULL;
   }
 
-  STableCheckInfo *pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, 0);
+  STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, 0);
 
   pTsdbReadHandle->pSchema = metaGetTbTSchema(pVnode->pMeta, pCheckInfo->tableId, 0);
-  int32_t numOfCols = taosArrayGetSize(pTsdbReadHandle->suppInfo.defaultLoadColumn);
+  int32_t  numOfCols = taosArrayGetSize(pTsdbReadHandle->suppInfo.defaultLoadColumn);
   int16_t* ids = pTsdbReadHandle->suppInfo.defaultLoadColumn->pData;
 
   STSchema* pSchema = pTsdbReadHandle->pSchema;
 
   int32_t i = 0, j = 0;
-  while(i < numOfCols && j < pSchema->numOfCols) {
+  while (i < numOfCols && j < pSchema->numOfCols) {
     if (ids[i] == pSchema->columns[j].colId) {
       pTsdbReadHandle->suppInfo.slotIds[i] = j;
       i++;
@@ -1136,7 +1165,7 @@ static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBl
                                    int32_t slotIndex) {
   int64_t st = taosGetTimestampUs();
 
-  int32_t   code = tdInitDataCols(pTsdbReadHandle->pDataCols, pTsdbReadHandle->pSchema);
+  int32_t code = tdInitDataCols(pTsdbReadHandle->pDataCols, pTsdbReadHandle->pSchema);
   if (code != TSDB_CODE_SUCCESS) {
     tsdbError("%p failed to malloc buf for pDataCols, %s", pTsdbReadHandle, pTsdbReadHandle->idStr);
     terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;

source/dnode/vnode/src/tsdb/tsdbSma.c

@@ -105,7 +105,7 @@ typedef struct {
 
 #define RSMA_TASK_INFO_HASH_SLOT 8
 struct SRSmaInfo {
-  void *taskInfo[TSDB_RSMA_RETENTION_2];  // qTaskInfo_t
+  void *taskInfo[TSDB_RETENTION_L2];  // qTaskInfo_t
 };
 
 struct SSmaStat {
@@ -127,7 +127,7 @@ static FORCE_INLINE void tsdbFreeTaskHandle(qTaskInfo_t *taskHandle) {
 }
 
 static FORCE_INLINE void *tsdbFreeRSmaInfo(SRSmaInfo *pInfo) {
-  for (int32_t i = 0; i < TSDB_RSMA_RETENTION_MAX; ++i) {
+  for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) {
     if (pInfo->taskInfo[i]) {
       tsdbFreeTaskHandle(pInfo->taskInfo[i]);
     }
@@ -175,7 +175,8 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg);
 static FORCE_INLINE int32_t tsdbUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
 static FORCE_INLINE int32_t tsdbUpdateTbUidListImpl(STsdb *pTsdb, tb_uid_t *suid, SArray *tbUids);
 static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, int32_t inputType,
-                                                qTaskInfo_t *taskInfo, tb_uid_t suid, int8_t retention);
+                                                qTaskInfo_t *taskInfo, STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid,
+                                                int8_t level);
 // mgmt interface
 static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid);
 
@@ -1976,6 +1977,7 @@ static int32_t tsdbFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) {
 
     if (!pBlock) break;
     tsdbUidStorePut(pStore, msgIter.suid, NULL);
+    pStore->uid = msgIter.uid;  // TODO: remove, just for debugging
   }
 
   if (terrno != TSDB_CODE_SUCCESS) return -1;
@@ -1983,13 +1985,15 @@ static int32_t tsdbFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) {
 }
 
 static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg, int32_t inputType,
-                                                qTaskInfo_t *taskInfo, tb_uid_t suid, int8_t retention) {
+                                                qTaskInfo_t *taskInfo, STSchema *pTSchema, tb_uid_t suid, tb_uid_t uid,
+                                                int8_t level) {
   SArray *pResult = NULL;
-  tsdbDebug("vgId:%d execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, REPO_ID(pTsdb), retention, taskInfo,
+  tsdbDebug("vgId:%d execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, REPO_ID(pTsdb), level, taskInfo,
             suid);
+
   qSetStreamInput(taskInfo, pMsg, inputType);
   while (1) {
-    SSDataBlock *output;
+    SSDataBlock *output = NULL;
     uint64_t     ts;
     if (qExecTask(taskInfo, &output, &ts) < 0) {
       ASSERT(false);
@@ -2010,22 +2014,36 @@ static FORCE_INLINE int32_t tsdbExecuteRSmaImpl(STsdb *pTsdb, const void *pMsg,
 
   if (taosArrayGetSize(pResult) > 0) {
     blockDebugShowData(pResult);
+    STsdb      *sinkTsdb = (level == TSDB_RETENTION_L1 ? pTsdb->pVnode->pRSma1 : pTsdb->pVnode->pRSma2);
+    SSubmitReq *pReq = NULL;
+    if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, TD_VID(pTsdb->pVnode), uid, suid) != 0) {
+      taosArrayDestroy(pResult);
+      return TSDB_CODE_FAILED;
+    }
+    if (tsdbProcessSubmitReq(sinkTsdb, INT64_MAX, pReq) != 0) {
+      taosArrayDestroy(pResult);
+      taosMemoryFreeClear(pReq);
+      return TSDB_CODE_FAILED;
+    }
+    taosMemoryFreeClear(pReq);
   } else {
-    tsdbWarn("vgId:%d no rsma_1 data generated since %s", REPO_ID(pTsdb), tstrerror(terrno));
+    tsdbWarn("vgId:%d no rsma % " PRIi8 " data generated since %s", REPO_ID(pTsdb), level, tstrerror(terrno));
   }
-  
+
   taosArrayDestroy(pResult);
 
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t tsdbExecuteRSma(STsdb *pTsdb, const void *pMsg, int32_t inputType, tb_uid_t suid) {
+static int32_t tsdbExecuteRSma(STsdb *pTsdb, const void *pMsg, int32_t inputType, tb_uid_t suid, tb_uid_t uid) {
   SSmaEnv *pEnv = REPO_RSMA_ENV(pTsdb);
   if (!pEnv) {
     // only applicable when rsma env exists
     return TSDB_CODE_SUCCESS;
   }
 
+  ASSERT(uid != 0);  // TODO: remove later
+
   SSmaStat  *pStat = SMA_ENV_STAT(pEnv);
   SRSmaInfo *pRSmaInfo = NULL;
 
@@ -2037,8 +2055,11 @@ int32_t tsdbExecuteRSma(STsdb *pTsdb, const void *pMsg, int32_t inputType, tb_ui
   }
 
   if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
-    tsdbExecuteRSmaImpl(pTsdb, pMsg, inputType, pRSmaInfo->taskInfo[0], suid, TSDB_RSMA_RETENTION_1);
-    tsdbExecuteRSmaImpl(pTsdb, pMsg, inputType, pRSmaInfo->taskInfo[1], suid, TSDB_RSMA_RETENTION_2);
+    // TODO: use the proper schema instead of 0, and cache STSchema in cache
+    STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, suid, 0);
+    tsdbExecuteRSmaImpl(pTsdb, pMsg, inputType, pRSmaInfo->taskInfo[0], pTSchema, suid, uid, TSDB_RETENTION_L1);
+    tsdbExecuteRSmaImpl(pTsdb, pMsg, inputType, pRSmaInfo->taskInfo[1], pTSchema, suid, uid, TSDB_RETENTION_L2);
+    taosMemoryFree(pTSchema);
   }
 
   return TSDB_CODE_SUCCESS;
@@ -2056,12 +2077,12 @@ int32_t tsdbTriggerRSma(STsdb *pTsdb, void *pMsg, int32_t inputType) {
     tsdbFetchSubmitReqSuids(pMsg, &uidStore);
 
     if (uidStore.suid != 0) {
-      tsdbExecuteRSma(pTsdb, pMsg, inputType, uidStore.suid);
+      tsdbExecuteRSma(pTsdb, pMsg, inputType, uidStore.suid, uidStore.uid);
 
       void *pIter = taosHashIterate(uidStore.uidHash, NULL);
       while (pIter) {
         tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL);
-        tsdbExecuteRSma(pTsdb, pMsg, inputType, *pTbSuid);
+        tsdbExecuteRSma(pTsdb, pMsg, inputType, *pTbSuid, 0);
         pIter = taosHashIterate(uidStore.uidHash, pIter);
       }
 

source/dnode/vnode/src/tsdb/tsdbTDBImpl.c

@@ -97,7 +97,7 @@ int32_t tsdbCloseDBF(SDBFile *pDBF) {
 int32_t tsdbSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) {
   int32_t ret;
 
-  ret = tdbDbPut(pDBF->pDB, pKey, keyLen, pVal, valLen, txn);
+  ret = tdbDbInsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn);
   if (ret < 0) {
     tsdbError("Failed to create insert sma data into db, ret = %d", ret);
     return -1;

source/dnode/vnode/src/vnd/vnodeCfg.c

@@ -66,7 +66,6 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
   if (tjsonAddIntegerToObject(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1;
-#ifdef TSDB_VNODE_SMA_DEBUG
   if (pCfg->tsdbCfg.retentions[0].freq > 0) {
     int32_t nRetention = 1;
     if (pCfg->tsdbCfg.retentions[1].freq > 0) {
@@ -87,7 +86,6 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
       tjsonAddItemToArray(pNodeRetentions, pNodeRetention);
     }
   }
-#endif
   if (tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1;
@@ -135,11 +133,11 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
   if (tjsonGetNumberValue(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1;
-#ifdef TSDB_VNODE_SMA_DEBUG
   SJson *pNodeRetentions = tjsonGetObjectItem(pJson, "retentions");
-  int    nRetention = tjsonGetArraySize(pNodeRetentions);
-  ASSERT(nRetention <= TSDB_RSMA_RETENTION_MAX);
-
+  int32_t nRetention = tjsonGetArraySize(pNodeRetentions);
+  if (nRetention > TSDB_RETENTION_MAX) {
+    nRetention = TSDB_RETENTION_MAX;
+  }
   for (int32_t i = 0; i < nRetention; ++i) {
     SJson *pNodeRetention = tjsonGetArrayItem(pNodeRetentions, i);
     ASSERT(pNodeRetention != NULL);
@@ -148,7 +146,6 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
     tjsonGetNumberValue(pNodeRetention, "keep", (pCfg->tsdbCfg.retentions)[i].keep);
     tjsonGetNumberValue(pNodeRetention, "keepUnit", (pCfg->tsdbCfg.retentions)[i].keepUnit);
   }
-#endif
   if (tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1;

source/dnode/vnode/src/vnd/vnodeCommit.c

@@ -47,9 +47,26 @@ int vnodeBegin(SVnode *pVnode) {
   }
 
   // begin tsdb
-  if (tsdbBegin(pVnode->pTsdb) < 0) {
-    vError("vgId:%d failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
-    return -1;
+  if (vnodeIsRollup(pVnode)) {
+    if (tsdbBegin(VND_RSMA0(pVnode)) < 0) {
+      vError("vgId:%d failed to begin rsma0 since %s", TD_VID(pVnode), tstrerror(terrno));
+      return -1;
+    }
+
+    if (tsdbBegin(VND_RSMA1(pVnode)) < 0) {
+      vError("vgId:%d failed to begin rsma1 since %s", TD_VID(pVnode), tstrerror(terrno));
+      return -1;
+    }
+
+    if (tsdbBegin(VND_RSMA2(pVnode)) < 0) {
+      vError("vgId:%d failed to begin rsma2 since %s", TD_VID(pVnode), tstrerror(terrno));
+      return -1;
+    }
+  } else {
+    if (tsdbBegin(pVnode->pTsdb) < 0) {
+      vError("vgId:%d failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
+      return -1;
+    }
   }
 
   return 0;
@@ -212,10 +229,28 @@ int vnodeCommit(SVnode *pVnode) {
     ASSERT(0);
     return -1;
   }
-  if (tsdbCommit(pVnode->pTsdb) < 0) {
-    ASSERT(0);
-    return -1;
+
+  if(vnodeIsRollup(pVnode)) {
+    if (tsdbCommit(VND_RSMA0(pVnode)) < 0) {
+      ASSERT(0);
+      return -1;
+    }
+    if (tsdbCommit(VND_RSMA1(pVnode)) < 0) {
+      ASSERT(0);
+      return -1;
+    }
+    if (tsdbCommit(VND_RSMA2(pVnode)) < 0) {
+      ASSERT(0);
+      return -1;
+    }
+  } else {
+    if (tsdbCommit(pVnode->pTsdb) < 0) {
+      ASSERT(0);
+      return -1;
+    }
   }
+
+  
   if (tqCommit(pVnode->pTq) < 0) {
     ASSERT(0);
     return -1;

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -445,14 +445,45 @@ static int vnodeProcessAlterTbReq(SVnode *pVnode, void *pReq, int32_t len, SRpcM
 }
 
 static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) {
-  SVDropTbReq req = {0};
-  SVDropTbReq rsp = {0};
+  SVDropTbBatchReq req = {0};
+  SVDropTbBatchRsp rsp = {0};
+  SCoder           coder = {0};
+  int              ret;
+
+  pRsp->msgType = TDMT_VND_CREATE_STB_RSP;
+  pRsp->pCont = NULL;
+  pRsp->contLen = 0;
+  pRsp->code = TSDB_CODE_SUCCESS;
 
   // decode req
+  tCoderInit(&coder, TD_LITTLE_ENDIAN, pReq, len, TD_DECODER);
+  ret = tDecodeSVDropTbBatchReq(&coder, &req);
+  if (ret < 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    pRsp->code = terrno;
+    goto _exit;
+  }
 
   // process req
+  rsp.pArray = taosArrayInit(sizeof(SVDropTbRsp), req.nReqs);
+  for (int iReq = 0; iReq < req.nReqs; iReq++) {
+    SVDropTbReq *pDropTbReq = req.pReqs + iReq;
+    SVDropTbRsp  dropTbRsp = {0};
 
-  // return rsp
+    /* code */
+    ret = metaDropTable(pVnode->pMeta, version, pDropTbReq);
+    if (ret < 0) {
+      dropTbRsp.code = TSDB_CODE_SUCCESS;
+    } else {
+      dropTbRsp.code = terrno;
+    }
+
+    taosArrayPush(rsp.pArray, &dropTbRsp);
+  }
+
+_exit:
+  tCoderClear(&coder);
+  // encode rsp (TODO)
   return 0;
 }
 
@@ -461,7 +492,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in
   SSubmitRsp  rsp = {0};
 
   pRsp->code = 0;
-  tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK);
+
   // handle the request
   if (tsdbInsertData(pVnode->pTsdb, version, pSubmitReq, &rsp) < 0) {
     pRsp->code = terrno;
@@ -470,12 +501,28 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in
 
   // pRsp->msgType = TDMT_VND_SUBMIT_RSP;
   // vnodeProcessSubmitReq(pVnode, ptr, pRsp);
-  // tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK);
 
   // encode the response (TODO)
   pRsp->pCont = rpcMallocCont(sizeof(SSubmitRsp));
   memcpy(pRsp->pCont, &rsp, sizeof(rsp));
   pRsp->contLen = sizeof(SSubmitRsp);
 
+  tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK);
+
   return 0;
 }
+
+int32_t tsdbProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) {
+  if (!pReq) {
+    terrno = TSDB_CODE_INVALID_PTR;
+    return TSDB_CODE_FAILED;
+  }
+
+  SSubmitReq *pSubmitReq = (SSubmitReq *)pReq;
+
+  if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) {
+    return TSDB_CODE_FAILED;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}

source/libs/executor/CMakeLists.txt

@@ -8,7 +8,7 @@ add_library(executor STATIC ${EXECUTOR_SRC})
 #        )
 
 target_link_libraries(executor
-            PRIVATE os util common function parser planner qcom vnode scalar nodes
+            PRIVATE os util common function parser planner qcom vnode scalar nodes index
         )
 
 target_include_directories(
@@ -19,4 +19,4 @@ target_include_directories(
 
 #if(${BUILD_TEST})
     ADD_SUBDIRECTORY(test)
-#endif(${BUILD_TEST})
+#endif(${BUILD_TEST})
\ No newline at end of file

source/libs/executor/src/indexoperator.c

@@ -15,7 +15,9 @@
 
 #include "indexoperator.h"
 #include "executorimpl.h"
+#include "index.h"
 #include "nodes.h"
+#include "tdatablock.h"
 
 typedef struct SIFCtx {
   int32_t   code;
@@ -48,11 +50,38 @@ typedef struct SIFCtx {
   } while (0)
 
 typedef struct SIFParam {
-  SArray *  result;
   SHashObj *pFilter;
+
+  SArray *result;
+  char *  condValue;
+
+  uint8_t  colValType;
+  col_id_t colId;
+  int64_t  suid;  // add later
+  char     dbName[TSDB_DB_NAME_LEN];
+  char     colName[TSDB_COL_NAME_LEN];
 } SIFParam;
 
-typedef int32_t (*sif_func_t)(SNode *left, SNode *rigth, SIFParam *output);
+static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) {
+  if (src == OP_TYPE_GREATER_THAN) {
+    *dst = QUERY_GREATER_THAN;
+  } else if (src == OP_TYPE_GREATER_EQUAL) {
+    *dst = QUERY_GREATER_EQUAL;
+  } else if (src == OP_TYPE_LOWER_THAN) {
+    *dst = QUERY_LESS_THAN;
+  } else if (src == OP_TYPE_LOWER_EQUAL) {
+    *dst = QUERY_LESS_EQUAL;
+  } else if (src == OP_TYPE_EQUAL) {
+    *dst = QUERY_TERM;
+  } else if (src == OP_TYPE_LIKE || src == OP_TYPE_MATCH || src == OP_TYPE_NMATCH) {
+    *dst = QUERY_REGEX;
+  } else {
+    return TSDB_CODE_QRY_INVALID_INPUT;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+typedef int32_t (*sif_func_t)(SIFParam *left, SIFParam *rigth, SIFParam *output);
 // construct tag filter operator later
 static void destroyTagFilterOperatorInfo(void *param) {
   STagFilterOperatorInfo *pInfo = (STagFilterOperatorInfo *)param;
@@ -60,7 +89,10 @@ static void destroyTagFilterOperatorInfo(void *param) {
 
 static void sifFreeParam(SIFParam *param) {
   if (param == NULL) return;
+
   taosArrayDestroy(param->result);
+  taosMemoryFree(param->condValue);
+  taosHashCleanup(param->pFilter);
 }
 
 static int32_t sifGetOperParamNum(EOperatorType ty) {
@@ -71,16 +103,72 @@ static int32_t sifGetOperParamNum(EOperatorType ty) {
   }
   return 2;
 }
+static int32_t sifValidateColumn(SColumnNode *cn) {
+  // add more check
+  if (cn == NULL) {
+    return TSDB_CODE_QRY_INVALID_INPUT;
+  }
+  if (cn->colType != COLUMN_TYPE_TAG) {
+    return TSDB_CODE_QRY_INVALID_INPUT;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t sifGetValueFromNode(SNode *node, char **value) {
+  // covert data From snode;
+  SValueNode *vn = (SValueNode *)node;
+
+  char *     pData = nodesGetValueFromNode(vn);
+  SDataType *pType = &vn->node.resType;
+  int32_t    type = pType->type;
+  int32_t    valLen = 0;
+
+  if (IS_VAR_DATA_TYPE(type)) {
+    int32_t dataLen = varDataTLen(pData);
+    if (type == TSDB_DATA_TYPE_JSON) {
+      if (*pData == TSDB_DATA_TYPE_NULL) {
+        dataLen = 0;
+      } else if (*pData == TSDB_DATA_TYPE_NCHAR) {
+        dataLen = varDataTLen(pData + CHAR_BYTES);
+      } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) {
+        dataLen = LONG_BYTES;
+      } else if (*pData == TSDB_DATA_TYPE_BOOL) {
+        dataLen = CHAR_BYTES;
+      }
+      dataLen += CHAR_BYTES;
+    }
+    valLen = dataLen;
+  } else {
+    valLen = pType->bytes;
+  }
+  char *tv = taosMemoryCalloc(1, valLen + 1);
+  if (tv == NULL) {
+    return TSDB_CODE_QRY_OUT_OF_MEMORY;
+  }
+
+  memcpy(tv, pData, valLen);
+  *value = tv;
+
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
   switch (nodeType(node)) {
     case QUERY_NODE_VALUE: {
       SValueNode *vn = (SValueNode *)node;
-
+      SIF_ERR_RET(sifGetValueFromNode(node, &param->condValue));
+      param->colId = -1;
       break;
     }
     case QUERY_NODE_COLUMN: {
       SColumnNode *cn = (SColumnNode *)node;
+      /*only support tag column*/
+      SIF_ERR_RET(sifValidateColumn(cn));
 
+      param->colId = cn->colId;
+      param->colValType = cn->node.resType.type;
+      memcpy(param->dbName, cn->dbName, sizeof(cn->dbName));
+      memcpy(param->colName, cn->colName, sizeof(cn->colName));
       break;
     }
     case QUERY_NODE_NODE_LIST: {
@@ -89,7 +177,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
         qError("invalid length for node:%p, length: %d", node, LIST_LENGTH(nl->pNodeList));
         SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
       }
-
+      SIF_ERR_RET(scalarGenerateSetFromList((void **)&param->pFilter, node, nl->dataType.type));
       if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) {
         taosHashCleanup(param->pFilter);
         qError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param));
@@ -163,58 +251,75 @@ static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *outpu
   qError("index-filter not support buildin function");
   return TSDB_CODE_QRY_INVALID_INPUT;
 }
+static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) {
+  SIndexTerm *tm = indexTermCreate(left->suid, DEFAULT, operType, left->colValType, left->colName,
+                                   strlen(left->colName), right->condValue, strlen(right->condValue));
+  if (tm == NULL) {
+    return TSDB_CODE_QRY_OUT_OF_MEMORY;
+  }
+  SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST);
 
-static int32_t sifLessThanFunc(SNode *left, SNode *rigth, SIFParam *output) {
-  // impl later
-  return TSDB_CODE_SUCCESS;
+  EIndexQueryType qtype = 0;
+  SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype));
+
+  indexMultiTermQueryAdd(mtm, tm, qtype);
+  int ret = indexSearch(NULL, mtm, output->result);
+  indexMultiTermQueryDestroy(mtm);
+  return ret;
 }
-static int32_t sifLessEqualFunc(SNode *left, SNode *rigth, SIFParam *output) {
-  // impl later
-  return TSDB_CODE_SUCCESS;
+
+static int32_t sifLessThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+  int id = OP_TYPE_LOWER_THAN;
+  return sifDoIndex(left, right, id, output);
 }
-static int32_t sifGreaterThanFunc(SNode *left, SNode *rigth, SIFParam *output) {
-  // impl later
-  return TSDB_CODE_SUCCESS;
+static int32_t sifLessEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+  int id = OP_TYPE_LOWER_EQUAL;
+  return sifDoIndex(left, right, id, output);
 }
-static int32_t sifGreaterEqualFunc(SNode *left, SNode *rigth, SIFParam *output) {
-  // impl later
-  return TSDB_CODE_SUCCESS;
+
+static int32_t sifGreaterThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+  int id = OP_TYPE_GREATER_THAN;
+  return sifDoIndex(left, right, id, output);
+}
+static int32_t sifGreaterEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+  int id = OP_TYPE_GREATER_EQUAL;
+  return sifDoIndex(left, right, id, output);
 }
 
-static int32_t sifEqualFunc(SNode *left, SNode *rigth, SIFParam *output) {
-  // impl later
-  return TSDB_CODE_SUCCESS;
+static int32_t sifEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+  int id = OP_TYPE_EQUAL;
+  return sifDoIndex(left, right, id, output);
 }
-static int32_t sifNotEqualFunc(SNode *left, SNode *rigth, SIFParam *output) {
-  // impl later
-  return TSDB_CODE_SUCCESS;
+static int32_t sifNotEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+  int id = OP_TYPE_NOT_EQUAL;
+  return sifDoIndex(left, right, id, output);
 }
-static int32_t sifInFunc(SNode *left, SNode *rigth, SIFParam *output) {
-  // impl later
-  return TSDB_CODE_SUCCESS;
+static int32_t sifInFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+  int id = OP_TYPE_IN;
+  return sifDoIndex(left, right, id, output);
 }
-static int32_t sifNotInFunc(SNode *left, SNode *right, SIFParam *output) {
-  // impl later
-  return TSDB_CODE_SUCCESS;
+static int32_t sifNotInFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+  int id = OP_TYPE_NOT_IN;
+  return sifDoIndex(left, right, id, output);
 }
-static int32_t sifLikeFunc(SNode *left, SNode *right, SIFParam *output) {
-  // impl later
-  return TSDB_CODE_SUCCESS;
+static int32_t sifLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+  int id = OP_TYPE_LIKE;
+  return sifDoIndex(left, right, id, output);
 }
-static int32_t sifNotLikeFunc(SNode *left, SNode *right, SIFParam *output) {
-  // impl later
-  return TSDB_CODE_SUCCESS;
+static int32_t sifNotLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+  int id = OP_TYPE_NOT_LIKE;
+  return sifDoIndex(left, right, id, output);
 }
 
-static int32_t sifMatchFunc(SNode *left, SNode *rigth, SIFParam *output) {
-  // impl later
-  return TSDB_CODE_SUCCESS;
+static int32_t sifMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+  int id = OP_TYPE_MATCH;
+  return sifDoIndex(left, right, id, output);
 }
-static int32_t sifNotMatchFunc(SNode *left, SNode *rigth, SIFParam *output) {
-  // impl later
-  return TSDB_CODE_SUCCESS;
+static int32_t sifNotMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
+  int id = OP_TYPE_NMATCH;
+  return sifDoIndex(left, right, id, output);
 }
-static int32_t sifDefaultFunc(SNode *left, SNode *rigth, SIFParam *output) {
+static int32_t sifDefaultFunc(SIFParam *left, SIFParam *right, SIFParam *output) {
   // add more except
   return TSDB_CODE_QRY_INVALID_INPUT;
 }
@@ -252,17 +357,18 @@ static sif_func_t sifGetOperFn(int32_t funcId) {
   return sifDefaultFunc;
 }
 static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) {
-  int32_t   code = 0;
-  SIFParam *params = NULL;
-  SIF_ERR_RET(sifInitOperParams(&params, node, ctx));
-
+  int32_t code = 0;
   int32_t nParam = sifGetOperParamNum(node->opType);
   if (nParam <= 1) {
     SIF_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
   }
+
+  SIFParam *params = NULL;
+  SIF_ERR_RET(sifInitOperParams(&params, node, ctx));
+
   sif_func_t operFn = sifGetOperFn(node->opType);
 
-  return operFn(node->pLeft, node->pRight, output);
+  return operFn(&params[0], nParam > 1 ? &params[1] : NULL, output);
 _return:
   taosMemoryFree(params);
   SIF_RET(code);
@@ -335,7 +441,6 @@ static EDealRes sifWalkOper(SNode *pNode, void *context) {
   if (ctx->code) {
     return DEAL_RES_ERROR;
   }
-
   if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) {
     ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY;
     return DEAL_RES_ERROR;
@@ -387,6 +492,7 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) {
     qError("index-filter failed to taosHashInit");
     return TSDB_CODE_QRY_OUT_OF_MEMORY;
   }
+
   nodesWalkExprPostOrder(pNode, sifCalcWalker, &ctx);
   SIF_ERR_RET(ctx.code);
 
@@ -425,6 +531,7 @@ SIdxFltStatus idxGetFltStatus(SNode *pFilterNode) {
   if (pFilterNode == NULL) {
     return SFLT_NOT_INDEX;
   }
+
   // impl later
   return SFLT_ACCURATE_INDEX;
 }

source/libs/function/src/builtins.c

@@ -913,7 +913,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "tbname",
     .type = FUNCTION_TYPE_TBNAME,
-    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC,
+    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_SCAN_PC_FUNC,
     .translateFunc = translateTbnameColumn,
     .getEnvFunc   = NULL,
     .initFunc     = NULL,

source/libs/index/inc/indexFst.h

@@ -52,7 +52,6 @@ typedef struct FstRange {
   uint64_t end;
 } FstRange;
 
-typedef enum { GE, GT, LE, LT } RangeType;
 typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal } State;
 typedef enum { Ordered, OutOfOrdered, DuplicateKey } OrderType;
 
@@ -174,9 +173,9 @@ Output   fstStateFinalOutput(FstState* state, uint64_t version, FstSlice* date,
 uint64_t fstStateFindInput(FstState* state, FstNode* node, uint8_t b, bool* null);
 
 #define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext)
-#define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans)
-#define FST_STATE_ANY_TRANS(node) (node->state.state == AnyTrans)
-#define FST_STATE_EMPTY_FINAL(node) (node->state.state == EmptyFinal)
+#define FST_STATE_ONE_TRNAS(node)      (node->state.state == OneTrans)
+#define FST_STATE_ANY_TRANS(node)      (node->state.state == AnyTrans)
+#define FST_STATE_EMPTY_FINAL(node)    (node->state.state == EmptyFinal)
 
 typedef struct FstLastTransition {
   uint8_t inp;

source/libs/index/inc/indexInt.h

@@ -34,6 +34,7 @@
 extern "C" {
 #endif
 
+typedef enum { LT, LE, GT, GE } RangeType;
 typedef enum { kTypeValue, kTypeDeletion } STermValueType;
 
 typedef struct SIndexStat {
@@ -57,7 +58,7 @@ struct SIndex {
 
   char* path;
 
-  SIndexStat      stat;
+  SIndexStat    stat;
   TdThreadMutex mtx;
 };
 
@@ -165,7 +166,7 @@ int32_t indexSerialCacheKey(ICacheKey* key, char* buf);
   } while (0)
 
 #define INDEX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0)
-#define INDEX_TYPE_GET_TYPE(ty) (ty & 0x0F)
+#define INDEX_TYPE_GET_TYPE(ty)                  (ty & 0x0F)
 #define INDEX_TYPE_ADD_EXTERN_TYPE(ty, exTy) \
   do {                                       \
     uint8_t oldTy = ty;                      \

source/libs/index/src/index.c

@@ -175,55 +175,19 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
   return 0;
 }
 int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result) {
-#ifdef USE_LUCENE
-  EIndexOperatorType opera = multiQuerys->opera;
-
-  int    nQuery = taosArrayGetSize(multiQuerys->query);
-  char** fields = taosMemoryMalloc(sizeof(char*) * nQuery);
-  char** keys = taosMemoryMalloc(sizeof(char*) * nQuery);
-  int*   types = taosMemoryMalloc(sizeof(int) * nQuery);
-
-  for (int i = 0; i < nQuery; i++) {
-    SIndexTermQuery* p = taosArrayGet(multiQuerys->query, i);
-    SIndexTerm*      term = p->field_value;
-
-    fields[i] = taosMemoryCalloc(1, term->nKey + 1);
-    keys[i] = taosMemoryCalloc(1, term->nVal + 1);
-
-    memcpy(fields[i], term->key, term->nKey);
-    memcpy(keys[i], term->val, term->nVal);
-    types[i] = (int)(p->type);
-  }
-  int* tResult = NULL;
-  int  tsz = 0;
-  index_multi_search(index->index, (const char**)fields, (const char**)keys, types, nQuery, opera, &tResult, &tsz);
-
-  for (int i = 0; i < tsz; i++) {
-    taosArrayPush(result, &tResult[i]);
-  }
-
-  for (int i = 0; i < nQuery; i++) {
-    taosMemoryFree(fields[i]);
-    taosMemoryFree(keys[i]);
-  }
-  taosMemoryFree(fields);
-  taosMemoryFree(keys);
-  taosMemoryFree(types);
-#endif
-
 #ifdef USE_INVERTED_INDEX
   EIndexOperatorType opera = multiQuerys->opera;  // relation of querys
 
-  SArray* interResults = taosArrayInit(4, POINTER_BYTES);
+  SArray* iRslts = taosArrayInit(4, POINTER_BYTES);
   int     nQuery = taosArrayGetSize(multiQuerys->query);
   for (size_t i = 0; i < nQuery; i++) {
-    SIndexTermQuery* qTerm = taosArrayGet(multiQuerys->query, i);
-    SArray*          tResult = NULL;
-    indexTermSearch(index, qTerm, &tResult);
-    taosArrayPush(interResults, (void*)&tResult);
+    SIndexTermQuery* qterm = taosArrayGet(multiQuerys->query, i);
+    SArray*          trslt = NULL;
+    indexTermSearch(index, qterm, &trslt);
+    taosArrayPush(iRslts, (void*)&trslt);
   }
-  indexMergeFinalResults(interResults, opera, result);
-  indexInterResultsDestroy(interResults);
+  indexMergeFinalResults(iRslts, opera, result);
+  indexInterResultsDestroy(iRslts);
 
 #endif
   return 0;
@@ -258,13 +222,13 @@ void indexOptsDestroy(SIndexOpts* opts) {
  *
  */
 SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) {
-  SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)taosMemoryMalloc(sizeof(SIndexMultiTermQuery));
-  if (p == NULL) {
+  SIndexMultiTermQuery* mtq = (SIndexMultiTermQuery*)taosMemoryMalloc(sizeof(SIndexMultiTermQuery));
+  if (mtq == NULL) {
     return NULL;
   }
-  p->opera = opera;
-  p->query = taosArrayInit(4, sizeof(SIndexTermQuery));
-  return p;
+  mtq->opera = opera;
+  mtq->query = taosArrayInit(4, sizeof(SIndexTermQuery));
+  return mtq;
 }
 void indexMultiTermQueryDestroy(SIndexMultiTermQuery* pQuery) {
   for (int i = 0; i < taosArrayGetSize(pQuery->query); i++) {
@@ -280,25 +244,26 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde
   return 0;
 }
 
-SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName,
-                            int32_t nColName, const char* colVal, int32_t nColVal) {
-  SIndexTerm* t = (SIndexTerm*)taosMemoryCalloc(1, (sizeof(SIndexTerm)));
-  if (t == NULL) {
+SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, int8_t queryType, uint8_t colType,
+                            const char* colName, int32_t nColName, const char* colVal, int32_t nColVal) {
+  SIndexTerm* tm = (SIndexTerm*)taosMemoryCalloc(1, (sizeof(SIndexTerm)));
+  if (tm == NULL) {
     return NULL;
   }
 
-  t->suid = suid;
-  t->operType = oper;
-  t->colType = colType;
+  tm->suid = suid;
+  tm->operType = oper;
+  tm->colType = colType;
+
+  tm->colName = (char*)taosMemoryCalloc(1, nColName + 1);
+  memcpy(tm->colName, colName, nColName);
+  tm->nColName = nColName;
 
-  t->colName = (char*)taosMemoryCalloc(1, nColName + 1);
-  memcpy(t->colName, colName, nColName);
-  t->nColName = nColName;
+  tm->colVal = (char*)taosMemoryCalloc(1, nColVal + 1);
+  memcpy(tm->colVal, colVal, nColVal);
+  tm->nColVal = nColVal;
 
-  t->colVal = (char*)taosMemoryCalloc(1, nColVal + 1);
-  memcpy(t->colVal, colVal, nColVal);
-  t->nColVal = nColVal;
-  return t;
+  return tm;
 }
 void indexTermDestroy(SIndexTerm* p) {
   taosMemoryFree(p->colName);

source/libs/index/src/indexCache.c

@@ -34,9 +34,129 @@ static char*   indexCacheTermGet(const void* pData);
 
 static MemTable* indexInternalCacheCreate(int8_t type);
 
+static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchSuffix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s);
+/*comm func of compare, used in (LE/LT/GE/GT compare)*/
+static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s,
+                                      RangeType type);
+
+typedef enum { MATCH, CONTINUE, BREAK } TExeCond;
+typedef TExeCond (*_cache_range_compare)(void* a, void* b, int8_t type);
+
+static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { return MATCH; }
+static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { return MATCH; }
+static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { return MATCH; }
+static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { return MATCH; }
+
+static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {tCompareLessThan, tCompareLessEqual,
+                                                                    tCompareGreaterThan, tCompareGreaterEqual};
+
+static int32_t (*cacheSearch[])(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) = {
+    cacheSearchTerm,      cacheSearchPrefix,      cacheSearchSuffix,       cacheSearchRegex, cacheSearchLessThan,
+    cacheSearchLessEqual, cacheSearchGreaterThan, cacheSearchGreaterEqual, cacheSearchRange};
+
 static void doMergeWork(SSchedMsg* msg);
 static bool indexCacheIteratorNext(Iterate* itera);
 
+static int32_t cacheSearchTerm(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+  if (cache == NULL) {
+    return 0;
+  }
+
+  MemTable* mem = cache;
+  char*     key = indexCacheTermGet(ct);
+
+  SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
+  while (tSkipListIterNext(iter)) {
+    SSkipListNode* node = tSkipListIterGet(iter);
+    if (node == NULL) {
+      break;
+    }
+    CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node);
+    if (0 == strcmp(c->colVal, ct->colVal)) {
+      if (c->operaType == ADD_VALUE) {
+        INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid)
+        // taosArrayPush(result, &c->uid);
+        *s = kTypeValue;
+      } else if (c->operaType == DEL_VALUE) {
+        INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid)
+      }
+    } else {
+      break;
+    }
+  }
+  tSkipListDestroyIter(iter);
+  return 0;
+}
+static int32_t cacheSearchPrefix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+  // impl later
+  return 0;
+}
+static int32_t cacheSearchSuffix(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+  // impl later
+  return 0;
+}
+static int32_t cacheSearchRegex(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+  // impl later
+  return 0;
+}
+static int32_t cacheSearchCompareFunc(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s,
+                                      RangeType type) {
+  if (cache == NULL) {
+    return 0;
+  }
+  _cache_range_compare cmpFn = rangeCompare[type];
+
+  MemTable* mem = cache;
+  char*     key = indexCacheTermGet(ct);
+
+  SSkipListIterator* iter = tSkipListCreateIter(mem->mem);
+  while (tSkipListIterNext(iter)) {
+    SSkipListNode* node = tSkipListIterGet(iter);
+    if (node == NULL) {
+      break;
+    }
+    CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node);
+    TExeCond   cond = cmpFn(c->colVal, ct->colVal, ct->colType);
+    if (cond == MATCH) {
+      if (c->operaType == ADD_VALUE) {
+        INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid)
+        // taosArrayPush(result, &c->uid);
+        *s = kTypeValue;
+      } else if (c->operaType == DEL_VALUE) {
+        INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid)
+      }
+    } else if (cond == CONTINUE) {
+    } else if (cond == BREAK) {
+      break;
+    }
+  }
+  tSkipListDestroyIter(iter);
+  return TSDB_CODE_SUCCESS;
+}
+static int32_t cacheSearchLessThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+  return cacheSearchCompareFunc(cache, ct, tr, s, LT);
+}
+static int32_t cacheSearchLessEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+  return cacheSearchCompareFunc(cache, ct, tr, s, LE);
+}
+static int32_t cacheSearchGreaterThan(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+  return cacheSearchCompareFunc(cache, ct, tr, s, GT);
+}
+static int32_t cacheSearchGreaterEqual(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+  return cacheSearchCompareFunc(cache, ct, tr, s, GE);
+}
+static int32_t cacheSearchRange(void* cache, CacheTerm* ct, SIdxTempResult* tr, STermValueType* s) {
+  // impl later
+  return 0;
+}
 static IterateValue* indexCacheIteratorGetValue(Iterate* iter);
 
 IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8_t type) {
@@ -263,33 +383,7 @@ static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SI
   if (mem == NULL) {
     return 0;
   }
-  char* key = indexCacheTermGet(ct);
-
-  SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
-  while (tSkipListIterNext(iter)) {
-    SSkipListNode* node = tSkipListIterGet(iter);
-    if (node != NULL) {
-      CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node);
-      if (qtype == QUERY_TERM) {
-        if (0 == strcmp(c->colVal, ct->colVal)) {
-          if (c->operaType == ADD_VALUE) {
-            INDEX_MERGE_ADD_DEL(tr->deled, tr->added, c->uid)
-            // taosArrayPush(result, &c->uid);
-            *s = kTypeValue;
-          } else if (c->operaType == DEL_VALUE) {
-            INDEX_MERGE_ADD_DEL(tr->added, tr->deled, c->uid)
-          }
-        } else {
-          break;
-        }
-      } else if (qtype == QUERY_PREFIX) {
-      } else if (qtype == QUERY_SUFFIX) {
-      } else if (qtype == QUERY_RANGE) {
-      }
-    }
-  }
-  tSkipListDestroyIter(iter);
-  return 0;
+  return cacheSearch[qtype](mem, ct, tr, s);
 }
 int indexCacheSearch(void* cache, SIndexTermQuery* query, SIdxTempResult* result, STermValueType* s) {
   int64_t st = taosGetTimestampUs();

source/libs/index/src/indexTfile.c

@@ -57,6 +57,24 @@ static int     tfileCompare(const void* a, const void* b);
 static int     tfileParseFileName(const char* filename, uint64_t* suid, char* col, int* version);
 static void    tfileGenFileName(char* filename, uint64_t suid, const char* col, int version);
 static void    tfileGenFileFullName(char* fullname, const char* path, uint64_t suid, const char* col, int32_t version);
+/*
+ * search from  tfile
+ */
+static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchLessEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchGreaterThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchGreaterEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr);
+
+static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType ctype);
+
+static int32_t (*tfSearch[])(void* reader, SIndexTerm* tem, SIdxTempResult* tr) = {
+    tfSearchTerm,      tfSearchPrefix,      tfSearchSuffix,       tfSearchRegex, tfSearchLessThan,
+    tfSearchLessEqual, tfSearchGreaterThan, tfSearchGreaterEqual, tfSearchRange};
 
 TFileCache* tfileCacheCreate(const char* path) {
   TFileCache* tcache = taosMemoryCalloc(1, sizeof(TFileCache));
@@ -183,59 +201,194 @@ void tfileReaderDestroy(TFileReader* reader) {
   writerCtxDestroy(reader->ctx, reader->remove);
   taosMemoryFree(reader);
 }
+static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+  bool     hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
+  int      ret = 0;
+  char*    p = tem->colVal;
+  uint64_t sz = tem->nColVal;
+  if (hasJson) {
+    p = indexPackJsonData(tem);
+    sz = strlen(p);
+  }
+  int64_t  st = taosGetTimestampUs();
+  FstSlice key = fstSliceCreate(p, sz);
+  uint64_t offset;
+  if (fstGet(((TFileReader*)reader)->fst, &key, &offset)) {
+    int64_t et = taosGetTimestampUs();
+    int64_t cost = et - st;
+    indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex, time cost: %" PRIu64 "us",
+              tem->suid, tem->colName, tem->colVal, cost);
+
+    ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total);
+    cost = taosGetTimestampUs() - et;
+    indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", tem->suid,
+              tem->colName, tem->colVal, cost);
+  }
+  if (hasJson) {
+    taosMemoryFree(p);
+  }
+  fstSliceDestroy(&key);
+  return 0;
+}
 
-int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTempResult* tr) {
-  SIndexTerm*     term = query->term;
-  bool            hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(term->colType, TSDB_DATA_TYPE_JSON);
-  EIndexQueryType qtype = query->qType;
+static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+  bool     hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
+  char*    p = tem->colVal;
+  uint64_t sz = tem->nColVal;
+  if (hasJson) {
+    p = indexPackJsonData(tem);
+    sz = strlen(p);
+  }
 
-  // SArray* result = taosArrayInit(16, sizeof(uint64_t));
-  int ret = -1;
-  // refactor to callback later
-  if (qtype == QUERY_TERM) {
-    uint64_t offset;
-    char*    p = term->colVal;
-    uint64_t sz = term->nColVal;
-    if (hasJson) {
-      p = indexPackJsonData(term);
-      sz = strlen(p);
-    }
-    int64_t  st = taosGetTimestampUs();
-    FstSlice key = fstSliceCreate(p, sz);
-    if (fstGet(reader->fst, &key, &offset)) {
-      int64_t et = taosGetTimestampUs();
-      int64_t cost = et - st;
-      indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex, time cost: %" PRIu64 "us",
-                term->suid, term->colName, term->colVal, cost);
-
-      ret = tfileReaderLoadTableIds(reader, offset, tr->total);
-      cost = taosGetTimestampUs() - et;
-      indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", term->suid,
-                term->colName, term->colVal, cost);
-    } else {
-      indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found table info in tindex", term->suid, term->colName,
-                term->colVal);
-    }
-    fstSliceDestroy(&key);
-    if (hasJson) {
-      taosMemoryFree(p);
+  SArray* offsets = taosArrayInit(16, sizeof(uint64_t));
+
+  AutomationCtx*         ctx = automCtxCreate((void*)p, AUTOMATION_PREFIX);
+  FstStreamBuilder*      sb = fstSearch(((TFileReader*)reader)->fst, ctx);
+  StreamWithState*       st = streamBuilderIntoStream(sb);
+  StreamWithStateResult* rt = NULL;
+  while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
+    taosArrayPush(offsets, &(rt->out.out));
+    swsResultDestroy(rt);
+  }
+  streamWithStateDestroy(st);
+  fstStreamBuilderDestroy(sb);
+
+  int32_t ret = 0;
+  for (int i = 0; i < taosArrayGetSize(offsets); i++) {
+    uint64_t offset = *(uint64_t*)taosArrayGet(offsets, i);
+    ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total);
+    if (ret != 0) {
+      indexError("failed to find target tablelist");
+      return TSDB_CODE_TDB_FILE_CORRUPTED;
     }
-  } else if (qtype == QUERY_PREFIX) {
-    // handle later
-    //
-  } else if (qtype == QUERY_SUFFIX) {
-    // handle later
-  } else if (qtype == QUERY_REGEX) {
-    // handle later
-  } else if (qtype == QUERY_RANGE) {
-    // handle later
   }
-  tfileReaderUnRef(reader);
+  if (hasJson) {
+    taosMemoryFree(p);
+  }
+  return 0;
+}
+static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+  bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
 
-  // taosArrayAddAll(tr->total, result);
-  // taosArrayDestroy(result);
+  int      ret = 0;
+  char*    p = tem->colVal;
+  uint64_t sz = tem->nColVal;
+  if (hasJson) {
+    p = indexPackJsonData(tem);
+    sz = strlen(p);
+  }
+  int64_t  st = taosGetTimestampUs();
+  FstSlice key = fstSliceCreate(p, sz);
+  /*impl later*/
+  if (hasJson) {
+    taosMemoryFree(p);
+  }
+  fstSliceDestroy(&key);
+  return 0;
+}
+static int32_t tfSearchRegex(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+  bool hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
+
+  int      ret = 0;
+  char*    p = tem->colVal;
+  uint64_t sz = tem->nColVal;
+  if (hasJson) {
+    p = indexPackJsonData(tem);
+    sz = strlen(p);
+  }
+  int64_t  st = taosGetTimestampUs();
+  FstSlice key = fstSliceCreate(p, sz);
+  /*impl later*/
+
+  if (hasJson) {
+    taosMemoryFree(p);
+  }
+  fstSliceDestroy(&key);
+  return 0;
+}
+
+static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTempResult* tr, RangeType type) {
+  bool     hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
+  int      ret = 0;
+  char*    p = tem->colVal;
+  uint64_t sz = tem->nColVal;
+  if (hasJson) {
+    p = indexPackJsonData(tem);
+    sz = strlen(p);
+  }
+  SArray* offsets = taosArrayInit(16, sizeof(uint64_t));
+
+  AutomationCtx*    ctx = automCtxCreate((void*)p, AUTOMATION_ALWAYS);
+  FstStreamBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx);
+
+  FstSlice h = fstSliceCreate((uint8_t*)p, sz);
+  fstStreamBuilderSetRange(sb, &h, type);
+  fstSliceDestroy(&h);
+
+  StreamWithState*       st = streamBuilderIntoStream(sb);
+  StreamWithStateResult* rt = NULL;
+  while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
+    taosArrayPush(offsets, &(rt->out.out));
+    swsResultDestroy(rt);
+  }
+  streamWithStateDestroy(st);
+  fstStreamBuilderDestroy(sb);
+  return TSDB_CODE_SUCCESS;
+}
+static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+  return tfSearchCompareFunc(reader, tem, tr, LT);
+}
+static int32_t tfSearchLessEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+  return tfSearchCompareFunc(reader, tem, tr, LE);
+}
+static int32_t tfSearchGreaterThan(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+  return tfSearchCompareFunc(reader, tem, tr, GT);
+}
+static int32_t tfSearchGreaterEqual(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+  return tfSearchCompareFunc(reader, tem, tr, GE);
+}
+static int32_t tfSearchRange(void* reader, SIndexTerm* tem, SIdxTempResult* tr) {
+  bool     hasJson = INDEX_TYPE_CONTAIN_EXTERN_TYPE(tem->colType, TSDB_DATA_TYPE_JSON);
+  int      ret = 0;
+  char*    p = tem->colVal;
+  uint64_t sz = tem->nColVal;
+  if (hasJson) {
+    p = indexPackJsonData(tem);
+    sz = strlen(p);
+  }
+  int64_t  st = taosGetTimestampUs();
+  FstSlice key = fstSliceCreate(p, sz);
+  // uint64_t offset;
+  // if (fstGet(((TFileReader*)reader)->fst, &key, &offset)) {
+  //  int64_t et = taosGetTimestampUs();
+  //  int64_t cost = et - st;
+  //  indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex, time cost: %" PRIu64 "us",
+  //            tem->suid, tem->colName, tem->colVal, cost);
+
+  //  ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total);
+  //  cost = taosGetTimestampUs() - et;
+  //  indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, load all table info, time cost: %" PRIu64 "us", tem->suid,
+  //            tem->colName, tem->colVal, cost);
+  //}
+  if (hasJson) {
+    taosMemoryFree(p);
+  }
+  fstSliceDestroy(&key);
+  return 0;
+}
 
-  return ret;
+int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTempResult* tr) {
+  SIndexTerm*     term = query->term;
+  EIndexQueryType qtype = query->qType;
+  if (qtype >= sizeof(tfSearch) / sizeof(tfSearch[0])) {
+    indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found table info in tindex", term->suid, term->colName,
+              term->colVal);
+    return -1;
+  } else {
+    return tfSearch[qtype](reader, term, tr);
+  }
+  tfileReaderUnRef(reader);
+  return 0;
 }
 
 TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const char* colName, uint8_t colType) {

source/libs/index/test/indexTests.cc

@@ -483,9 +483,9 @@ TEST_F(IndexTFileEnv, test_tfile_write) {
 
   std::string     colName("voltage");
   std::string     colVal("ab");
-  SIndexTerm*     term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+  SIndexTerm*     term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                      colVal.c_str(), colVal.size());
-  SIndexTermQuery query = { term, QUERY_TERM};
+  SIndexTermQuery query = {term, QUERY_TERM};
 
   SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t));
   fObj->Get(&query, result);
@@ -557,7 +557,7 @@ TEST_F(IndexCacheEnv, cache_test) {
   std::string colName("voltage");
   {
     std::string colVal("v1");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
     indexTermDestroy(term);
@@ -565,28 +565,28 @@ TEST_F(IndexCacheEnv, cache_test) {
   }
   {
     std::string colVal("v3");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
     indexTermDestroy(term);
   }
   {
     std::string colVal("v2");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
     indexTermDestroy(term);
   }
   {
     std::string colVal("v3");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
     indexTermDestroy(term);
   }
   {
     std::string colVal("v3");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     coj->Put(term, colId, version++, suid++);
     indexTermDestroy(term);
@@ -595,14 +595,14 @@ TEST_F(IndexCacheEnv, cache_test) {
   std::cout << "--------first----------" << std::endl;
   {
     std::string colVal("v3");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     coj->Put(term, othColId, version++, suid++);
     indexTermDestroy(term);
   }
   {
     std::string colVal("v4");
-    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     coj->Put(term, othColId, version++, suid++);
     indexTermDestroy(term);
@@ -613,7 +613,7 @@ TEST_F(IndexCacheEnv, cache_test) {
     std::string colVal("v4");
     for (size_t i = 0; i < 10; i++) {
       colVal[colVal.size() - 1] = 'a' + i;
-      SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+      SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                          colVal.c_str(), colVal.size());
       coj->Put(term, colId, version++, suid++);
       indexTermDestroy(term);
@@ -623,9 +623,9 @@ TEST_F(IndexCacheEnv, cache_test) {
   // begin query
   {
     std::string     colVal("v3");
-    SIndexTerm*     term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*     term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
-    SIndexTermQuery query = { term, QUERY_TERM };
+    SIndexTermQuery query = {term, QUERY_TERM};
     SArray*         ret = (SArray*)taosArrayInit(4, sizeof(suid));
     STermValueType  valType;
 
@@ -638,9 +638,9 @@ TEST_F(IndexCacheEnv, cache_test) {
   }
   {
     std::string     colVal("v2");
-    SIndexTerm*     term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*     term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
-    SIndexTermQuery query = { term, QUERY_TERM };
+    SIndexTermQuery query = {term, QUERY_TERM};
     SArray*         ret = (SArray*)taosArrayInit(4, sizeof(suid));
     STermValueType  valType;
 
@@ -670,7 +670,7 @@ class IndexObj {
     return ret;
   }
   void Del(const std::string& colName, const std::string& colVal, uint64_t uid) {
-    SIndexTerm*      term = indexTermCreate(0, DEL_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreate(0, DEL_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -679,7 +679,7 @@ class IndexObj {
   }
   int WriteMillonData(const std::string& colName, const std::string& colVal = "Hello world",
                       size_t numOfTable = 100 * 10000) {
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -701,7 +701,7 @@ class IndexObj {
         // opt
         tColVal[taosRand() % colValSize] = 'a' + k % 26;
       }
-      SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+      SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                          tColVal.c_str(), tColVal.size());
       SIndexMultiTerm* terms = indexMultiTermCreate();
       indexMultiTermAdd(terms, term);
@@ -737,7 +737,7 @@ class IndexObj {
 
   int SearchOne(const std::string& colName, const std::string& colVal) {
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     indexMultiTermQueryAdd(mq, term, QUERY_TERM);
 
@@ -759,7 +759,7 @@ class IndexObj {
   }
   int SearchOneTarget(const std::string& colName, const std::string& colVal, uint64_t val) {
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     indexMultiTermQueryAdd(mq, term, QUERY_TERM);
 
@@ -784,7 +784,7 @@ class IndexObj {
 
   void PutOne(const std::string& colName, const std::string& colVal) {
     SIndexMultiTerm* terms = indexMultiTermCreate();
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     indexMultiTermAdd(terms, term);
     Put(terms, 10);
@@ -792,7 +792,7 @@ class IndexObj {
   }
   void PutOneTarge(const std::string& colName, const std::string& colVal, uint64_t val) {
     SIndexMultiTerm* terms = indexMultiTermCreate();
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     indexMultiTermAdd(terms, term);
     Put(terms, val);
@@ -832,7 +832,7 @@ TEST_F(IndexEnv2, testIndexOpen) {
   {
     std::string colName("tag1"), colVal("Hello");
 
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -847,7 +847,7 @@ TEST_F(IndexEnv2, testIndexOpen) {
     size_t      size = 200;
     std::string colName("tag1"), colVal("hello");
 
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -862,7 +862,7 @@ TEST_F(IndexEnv2, testIndexOpen) {
     size_t      size = 200;
     std::string colName("tag1"), colVal("Hello");
 
-    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*      term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     SIndexMultiTerm* terms = indexMultiTermCreate();
     indexMultiTermAdd(terms, term);
@@ -877,7 +877,7 @@ TEST_F(IndexEnv2, testIndexOpen) {
   {
     std::string           colName("tag1"), colVal("Hello");
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(0, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
     indexMultiTermQueryAdd(mq, term, QUERY_TERM);
 

source/libs/index/test/jsonUT.cc

@@ -40,7 +40,7 @@ TEST_F(JsonEnv, testWrite) {
   {
     std::string colName("test");
     std::string colVal("ab");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -53,7 +53,7 @@ TEST_F(JsonEnv, testWrite) {
   {
     std::string colName("voltage");
     std::string colVal("ab1");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -66,7 +66,7 @@ TEST_F(JsonEnv, testWrite) {
   {
     std::string colName("voltage");
     std::string colVal("123");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -81,7 +81,7 @@ TEST_F(JsonEnv, testWrite) {
     std::string colVal("ab");
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                     colVal.c_str(), colVal.size());
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));
@@ -95,7 +95,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
   {
     std::string colName("test");
     std::string colVal("ab");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -110,7 +110,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
     std::string colVal("abxxxxxxxxxxxx");
     for (int i = 0; i < 1000; i++) {
       colVal[i % colVal.size()] = '0' + i % 128;
-      SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+      SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                          colVal.c_str(), colVal.size());
 
       SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -124,7 +124,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
   {
     std::string colName("voltagefdadfa");
     std::string colVal("abxxxxxxxxxxxx");
-    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm* term = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                        colVal.c_str(), colVal.size());
 
     SIndexMultiTerm* terms = indexMultiTermCreate();
@@ -139,7 +139,7 @@ TEST_F(JsonEnv, testWriteMillonData) {
     std::string colVal("ab");
 
     SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST);
-    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+    SIndexTerm*           q = indexTermCreate(1, ADD_VALUE, 0, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
                                     colVal.c_str(), colVal.size());
 
     SArray* result = taosArrayInit(1, sizeof(uint64_t));

source/libs/parser/test/mockCatalog.cpp

@@ -100,6 +100,17 @@ void generateInformationSchema(MockCatalogService* mcs) {
   }
 }
 
+/*
+ * Table:t1
+ *        Field        |        Type        |      DataType      |  Bytes   |
+ * ==========================================================================
+ *          ts         |       column       |     TIMESTAMP      |    8     |
+ *          c1         |       column       |        INT         |    4     |
+ *          c2         |       column       |      VARCHAR       |    20    |
+ *          c3         |       column       |       BIGINT       |    8     |
+ *          c4         |       column       |       DOUBLE       |    8     |
+ *          c5         |       column       |       DOUBLE       |    8     |
+ */
 void generateTestT1(MockCatalogService* mcs) {
   ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 6)
                                .setPrecision(TSDB_TIME_PRECISION_MILLI)
@@ -113,6 +124,17 @@ void generateTestT1(MockCatalogService* mcs) {
   builder.done();
 }
 
+/*
+ * Super Table: st1
+ *        Field        |        Type        |      DataType      |  Bytes   |
+ * ==========================================================================
+ *          ts         |       column       |     TIMESTAMP      |    8     |
+ *          c1         |       column       |        INT         |    4     |
+ *          c2         |       column       |      VARCHAR       |    20    |
+ *         tag1        |        tag         |        INT         |    4     |
+ *         tag2        |        tag         |      VARCHAR       |    20    |
+ * Child Table: st1s1, st1s2
+ */
 void generateTestST1(MockCatalogService* mcs) {
   ITableBuilder& builder = mcs->createTableBuilder("test", "st1", TSDB_SUPER_TABLE, 3, 2)
                                .setPrecision(TSDB_TIME_PRECISION_MILLI)

source/libs/planner/test/planSTableTest.cpp

@@ -23,4 +23,6 @@ TEST_F(PlanSuperTableTest, tbname) {
   useDb("root", "test");
 
   run("select tbname from st1");
+
+  run("select tbname, tag1, tag2 from st1");
 }

source/libs/planner/test/planTestMain.cpp

@@ -36,11 +36,11 @@ class PlannerEnv : public testing::Environment {
 static void parseArg(int argc, char* argv[]) {
   int                  opt = 0;
   const char*          optstring = "";
-  static struct option long_options[] = {{"dump", no_argument, NULL, 'd'}, {0, 0, 0, 0}};
+  static struct option long_options[] = {{"dump", optional_argument, NULL, 'd'}, {0, 0, 0, 0}};
   while ((opt = getopt_long(argc, argv, optstring, long_options, NULL)) != -1) {
     switch (opt) {
       case 'd':
-        g_isDump = true;
+        setDumpModule(optarg);
         break;
       default:
         break;

source/libs/planner/test/planTestUtil.cpp

@@ -34,7 +34,41 @@ using namespace testing;
     }                                                                                                              \
   } while (0);
 
-bool g_isDump = false;
+enum DumpModule {
+  DUMP_MODULE_NOTHING = 1,
+  DUMP_MODULE_PARSER,
+  DUMP_MODULE_LOGIC,
+  DUMP_MODULE_OPTIMIZED,
+  DUMP_MODULE_SPLIT,
+  DUMP_MODULE_SCALED,
+  DUMP_MODULE_PHYSICAL,
+  DUMP_MODULE_SUBPLAN,
+  DUMP_MODULE_ALL
+};
+
+DumpModule g_dumpModule = DUMP_MODULE_NOTHING;
+
+void setDumpModule(const char* pModule) {
+  if (NULL == pModule) {
+    g_dumpModule = DUMP_MODULE_ALL;
+  } else if (0 == strncasecmp(pModule, "parser", strlen(pModule))) {
+    g_dumpModule = DUMP_MODULE_PARSER;
+  } else if (0 == strncasecmp(pModule, "logic", strlen(pModule))) {
+    g_dumpModule = DUMP_MODULE_LOGIC;
+  } else if (0 == strncasecmp(pModule, "optimized", strlen(pModule))) {
+    g_dumpModule = DUMP_MODULE_OPTIMIZED;
+  } else if (0 == strncasecmp(pModule, "split", strlen(pModule))) {
+    g_dumpModule = DUMP_MODULE_SPLIT;
+  } else if (0 == strncasecmp(pModule, "scaled", strlen(pModule))) {
+    g_dumpModule = DUMP_MODULE_SCALED;
+  } else if (0 == strncasecmp(pModule, "physical", strlen(pModule))) {
+    g_dumpModule = DUMP_MODULE_PHYSICAL;
+  } else if (0 == strncasecmp(pModule, "subplan", strlen(pModule))) {
+    g_dumpModule = DUMP_MODULE_SUBPLAN;
+  } else if (0 == strncasecmp(pModule, "all", strlen(pModule))) {
+    g_dumpModule = DUMP_MODULE_PHYSICAL;
+  }
+}
 
 class PlannerTestBaseImpl {
  public:
@@ -66,11 +100,9 @@ class PlannerTestBaseImpl {
       SQueryPlan* pPlan = nullptr;
       doCreatePhysiPlan(&cxt, pLogicPlan, &pPlan);
 
-      if (g_isDump) {
-        dump();
-      }
+      dump(g_dumpModule);
     } catch (...) {
-      dump();
+      dump(DUMP_MODULE_ALL);
       throw;
     }
   }
@@ -109,23 +141,48 @@ class PlannerTestBaseImpl {
     res_.physiSubplans_.clear();
   }
 
-  void dump() {
+  void dump(DumpModule module) {
+    if (DUMP_MODULE_NOTHING == module) {
+      return;
+    }
+
     cout << "==========================================sql : [" << stmtEnv_.sql_ << "]" << endl;
-    cout << "syntax tree : " << endl;
-    cout << res_.ast_ << endl;
-    cout << "raw logic plan : " << endl;
-    cout << res_.rawLogicPlan_ << endl;
-    cout << "optimized logic plan : " << endl;
-    cout << res_.optimizedLogicPlan_ << endl;
-    cout << "split logic plan : " << endl;
-    cout << res_.splitLogicPlan_ << endl;
-    cout << "scaled logic plan : " << endl;
-    cout << res_.scaledLogicPlan_ << endl;
-    cout << "physical plan : " << endl;
-    cout << res_.physiPlan_ << endl;
-    cout << "physical subplan : " << endl;
-    for (const auto& subplan : res_.physiSubplans_) {
-      cout << subplan << endl;
+
+    if (DUMP_MODULE_ALL == module || DUMP_MODULE_PARSER == module) {
+      cout << "syntax tree : " << endl;
+      cout << res_.ast_ << endl;
+    }
+
+    if (DUMP_MODULE_ALL == module || DUMP_MODULE_LOGIC == module) {
+      cout << "raw logic plan : " << endl;
+      cout << res_.rawLogicPlan_ << endl;
+    }
+
+    if (DUMP_MODULE_ALL == module || DUMP_MODULE_OPTIMIZED == module) {
+      cout << "optimized logic plan : " << endl;
+      cout << res_.optimizedLogicPlan_ << endl;
+    }
+
+    if (DUMP_MODULE_ALL == module || DUMP_MODULE_SPLIT == module) {
+      cout << "split logic plan : " << endl;
+      cout << res_.splitLogicPlan_ << endl;
+    }
+
+    if (DUMP_MODULE_ALL == module || DUMP_MODULE_SCALED == module) {
+      cout << "scaled logic plan : " << endl;
+      cout << res_.scaledLogicPlan_ << endl;
+    }
+
+    if (DUMP_MODULE_ALL == module || DUMP_MODULE_PHYSICAL == module) {
+      cout << "physical plan : " << endl;
+      cout << res_.physiPlan_ << endl;
+    }
+
+    if (DUMP_MODULE_ALL == module || DUMP_MODULE_SUBPLAN == module) {
+      cout << "physical subplan : " << endl;
+      for (const auto& subplan : res_.physiSubplans_) {
+        cout << subplan << endl;
+      }
     }
   }
 

source/libs/planner/test/planTestUtil.h

@@ -32,6 +32,6 @@ class PlannerTestBase : public testing::Test {
   std::unique_ptr<PlannerTestBaseImpl> impl_;
 };
 
-extern bool g_isDump;
+extern void setDumpModule(const char* pModule);
 
 #endif  // PLAN_TEST_UTIL_H

source/libs/scalar/src/sclvector.c

@@ -297,6 +297,22 @@ static FORCE_INLINE void varToNchar(char* buf, SScalarParam* pOut, int32_t rowIn
   taosMemoryFree(t);
 }
 
+static FORCE_INLINE void ncharToVar(char* buf, SScalarParam* pOut, int32_t rowIndex) {
+  int32_t inputLen = varDataLen(buf);
+
+  char* t = taosMemoryCalloc(1, inputLen + VARSTR_HEADER_SIZE);
+  int32_t len  = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t));
+  if (len < 0) {
+    taosMemoryFree(t);
+    return;
+  }
+  varDataSetLen(t, len);
+
+  colDataAppend(pOut->columnData, rowIndex, t, false);
+  taosMemoryFree(t);
+}
+
+
 //TODO opt performance, tmp is not needed.
 int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) {
   int32_t bufSize = pIn->columnData->info.bytes;
@@ -313,6 +329,10 @@ int32_t vectorConvertFromVarData(const SScalarParam* pIn, SScalarParam* pOut, in
     func = varToUnsigned;
   } else if (IS_FLOAT_TYPE(outType)) {
     func = varToFloat;
+  } else if (outType == TSDB_DATA_TYPE_BINARY) {   // nchar -> binary
+    ASSERT(inType == TSDB_DATA_TYPE_NCHAR);
+    func = ncharToVar;
+    vton = true;
   } else if (outType == TSDB_DATA_TYPE_NCHAR) {   // binary -> nchar
     ASSERT(inType == TSDB_DATA_TYPE_VARCHAR);
     func = varToNchar;
@@ -608,7 +628,7 @@ int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB+1][TSDB_DATA_TYPE_BLOB+1] = {
 /*BIGI*/   0,   0,   0,   0,   0,   0,   6,   7,   7,   0,   7,   5,   5,   5,   7,   0,   7,   0,   0,
 /*FLOA*/   0,   0,   0,   0,   0,   0,   0,   7,   7,   6,   7,   6,   6,   6,   6,   0,   7,   0,   0,
 /*DOUB*/   0,   0,   0,   0,   0,   0,   0,   0,   7,   7,   7,   7,   7,   7,   7,   0,   7,   0,   0,
-/*VARC*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   9,   0,   7,   7,   7,   7,   0,   0,   0,   0,
+/*VARC*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   9,   8,   7,   7,   7,   7,   0,   0,   0,   0,
 /*TIME*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   9,   9,   9,   9,   7,   0,   7,   0,   0,
 /*NCHA*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   7,   7,   7,   7,   0,   0,   0,   0,
 /*UTIN*/   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   12,  13,  14,  0,   7,   0,   0,

source/libs/tdb/inc/tdb.h

@@ -40,7 +40,9 @@ int tdbCommit(TENV *pEnv, TXN *pTxn);
 int tdbDbOpen(const char *fname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TENV *pEnv, TDB **ppDb);
 int tdbDbClose(TDB *pDb);
 int tdbDbDrop(TDB *pDb);
-int tdbDbPut(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn);
+int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn);
+int tdbDbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn);
+int tdbDbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn);
 int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen);
 int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen);
 
@@ -53,11 +55,9 @@ int tdbDbcMoveToLast(TDBC *pDbc);
 int tdbDbcMoveToNext(TDBC *pDbc);
 int tdbDbcMoveToPrev(TDBC *pDbc);
 int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, int *pvLen);
-
-int tdbDbcPut(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen);
-int tdbDbcUpdate(TDBC *pDbc, const void *pKey, int kLen, const void *pVal, int vLen);
-int tdbDbcDrop(TDBC *pDbc);
+int tdbDbcDelete(TDBC *pDbc);
 int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen);
+int tdbDbcUpsert(TDBC *pDbc, const void *pKey, int nKey, const void *pData, int nData, int insert);
 
 // TXN
 #define TDB_TXN_WRITE            0x1

source/libs/tdb/src/db/tdbDb.c

@@ -75,10 +75,16 @@ int tdbDbDrop(TDB *pDb) {
   return 0;
 }
 
-int tdbDbPut(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) {
+int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) {
   return tdbBtreeInsert(pDb->pBt, pKey, keyLen, pVal, valLen, pTxn);
 }
 
+int tdbDbDelete(TDB *pDb, const void *pKey, int kLen, TXN *pTxn) { return tdbBtreeDelete(pDb->pBt, pKey, kLen, pTxn); }
+
+int tdbDbUpsert(TDB *pDb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) {
+  return tdbBtreeUpsert(pDb->pBt, pKey, kLen, pVal, vLen, pTxn);
+}
+
 int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) {
   return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen);
 }
@@ -117,28 +123,16 @@ int tdbDbcGet(TDBC *pDbc, const void **ppKey, int *pkLen, const void **ppVal, in
   return tdbBtcGet(&pDbc->btc, ppKey, pkLen, ppVal, pvLen);
 }
 
-int tdbDbcPut(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen) {
-  // TODO
-  ASSERT(0);
-  return 0;
-}
-
-int tdbDbcUpdate(TDBC *pDbc, const void *pKey, int kLen, const void *pVal, int vLen) {
-  // TODO
-  ASSERT(0);
-  return 0;
-}
-
-int tdbDbcDrop(TDBC *pDbc) {
-  // TODO
-  ASSERT(0);
-  return 0;
-}
+int tdbDbcDelete(TDBC *pDbc) { return tdbBtcDelete(&pDbc->btc); }
 
 int tdbDbcNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen) {
   return tdbBtreeNext(&pDbc->btc, ppKey, kLen, ppVal, vLen);
 }
 
+int tdbDbcUpsert(TDBC *pDbc, const void *pKey, int nKey, const void *pData, int nData, int insert) {
+  return tdbBtcUpsert(&pDbc->btc, pKey, nKey, pData, nData, insert);
+}
+
 int tdbDbcClose(TDBC *pDbc) {
   if (pDbc) {
     tdbBtcClose(&pDbc->btc);

source/libs/tdb/src/db/tdbPage.c

@@ -171,6 +171,11 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl
   return 0;
 }
 
+int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell) {
+  tdbPageDropCell(pPage, idx);
+  return tdbPageInsertCell(pPage, idx, pCell, szCell, 0);
+}
+
 int tdbPageDropCell(SPage *pPage, int idx) {
   int    lidx;
   SCell *pCell;

source/libs/tdb/src/inc/tdbInt.h

@@ -128,6 +128,8 @@ struct SBTC {
 int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, tdb_cmpr_fn_t kcmpr, SBTree **ppBt);
 int tdbBtreeClose(SBTree *pBt);
 int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn);
+int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn);
+int tdbBtreeUpsert(SBTree *pBt, const void *pKey, int nKey, const void *pData, int nData, TXN *pTxn);
 int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen);
 int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen);
 
@@ -141,6 +143,8 @@ int tdbBtcMoveToNext(SBTC *pBtc);
 int tdbBtcMoveToPrev(SBTC *pBtc);
 int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen);
 int tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int *vLen);
+int tdbBtcDelete(SBTC *pBtc);
+int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int nData, int insert);
 
 // tdbPager.c ====================================
 
@@ -278,6 +282,7 @@ void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell
 void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *));
 int  tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl);
 int  tdbPageDropCell(SPage *pPage, int idx);
+int  tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell);
 void tdbPageCopy(SPage *pFromPage, SPage *pToPage);
 int  tdbPageCapacity(int pageSize, int amHdrSize);
 

source/libs/transport/src/transCli.c

@@ -949,9 +949,14 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
   }
 
   if (pCtx->pSem != NULL) {
-    tTrace("%s cli conn %p handle resp", pTransInst->label, pConn);
-    memcpy((char*)pCtx->pRsp, (char*)pResp, sizeof(*pResp));
+    tTrace("%s cli conn %p(sync) handle resp", pTransInst->label, pConn);
+    if (pCtx->pRsp == NULL) {
+      tTrace("%s cli conn %p(sync) failed to resp, ignore", pTransInst->label, pConn);
+    } else {
+      memcpy((char*)pCtx->pRsp, (char*)pResp, sizeof(*pResp));
+    }
     tsem_post(pCtx->pSem);
+    pCtx->pRsp = NULL;
   } else {
     tTrace("%s cli conn %p handle resp", pTransInst->label, pConn);
     pTransInst->cfp(pTransInst->parent, pResp, pEpSet);

source/util/src/terror.c

@@ -271,7 +271,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE,    "Invalid func retrieve
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST,      "Transaction already exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NOT_EXIST,          "Transaction not exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_INVALID_STAGE,      "Invalid stage to kill")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CANT_PARALLEL,      "Invalid stage to kill")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CONFLICT,           "Conflict transaction not completed")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_UNKNOW_ERROR,       "Unknown transaction error")
 
 // mnode-mq
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_ALREADY_EXIST,      "Topic already exists")

taos-tools @ 2f3dfddd

-Subproject commit bf6c766986c61ff4fc80421fdea682a8fd4b5b32
+Subproject commit 2f3dfddd4d9a869e706ba3cf98fb6d769404cd7c