Merge branch 'dev' into index

examples/c/tmq.c

@@ -28,15 +28,23 @@ static void msg_process(TAOS_RES* msg) {
   printf("db: %s\n", tmq_get_db_name(msg));
   printf("vg: %d\n", tmq_get_vgroup_id(msg));
   if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) {
-    void*   meta;
-    int32_t metaLen;
-    tmq_get_raw_meta(msg, &meta, &metaLen);
+    tmq_raw_data *raw = tmq_get_raw_meta(msg);
+    if(raw){
+      TAOS* pConn = taos_connect("192.168.1.86", "root", "taosdata", "abc1", 0);
+      if (pConn == NULL) {
+        return;
+      }
+      int32_t ret = taos_write_raw_meta(pConn, raw);
+      printf("write raw data: %s\n", tmq_err2str(ret));
+      free(raw);
+      taos_close(pConn);
+    }
     char* result = tmq_get_json_meta(msg);
     if(result){
       printf("meta result: %s\n", result);
       free(result);
     }
-    printf("meta, len is %d\n", metaLen);
+    printf("meta:%p\n", raw);
     return;
   }
   while (1) {

include/client/taos.h

@@ -260,15 +260,16 @@ enum tmq_res_t {
 };
 
 typedef enum tmq_res_t tmq_res_t;
-
-DLL_EXPORT tmq_res_t   tmq_get_res_type(TAOS_RES *res);
-DLL_EXPORT int32_t     tmq_get_raw_meta(TAOS_RES *res, void **raw_meta, int32_t *raw_meta_len);
-DLL_EXPORT int32_t     taos_write_raw_meta(TAOS *res, void *raw_meta, int32_t raw_meta_len);
-DLL_EXPORT char       *tmq_get_json_meta(TAOS_RES *res);   // Returning null means error. Returned result need to be freed.
-DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res);
-DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res);
-DLL_EXPORT int32_t     tmq_get_vgroup_id(TAOS_RES *res);
-DLL_EXPORT const char *tmq_get_table_name(TAOS_RES *res);
+typedef struct tmq_raw_data tmq_raw_data;
+
+DLL_EXPORT tmq_res_t     tmq_get_res_type(TAOS_RES *res);
+DLL_EXPORT tmq_raw_data *tmq_get_raw_meta(TAOS_RES *res);
+DLL_EXPORT int32_t       taos_write_raw_meta(TAOS *taos, tmq_raw_data *raw_meta);
+DLL_EXPORT char         *tmq_get_json_meta(TAOS_RES *res);   // Returning null means error. Returned result need to be freed.
+DLL_EXPORT const char   *tmq_get_topic_name(TAOS_RES *res);
+DLL_EXPORT const char   *tmq_get_db_name(TAOS_RES *res);
+DLL_EXPORT int32_t       tmq_get_vgroup_id(TAOS_RES *res);
+DLL_EXPORT const char   *tmq_get_table_name(TAOS_RES *res);
 
 /* ------------------------------ TMQ END -------------------------------- */
 

include/common/tcommon.h

@@ -55,7 +55,8 @@ enum {
 enum {
   STREAM_INPUT__DATA_SUBMIT = 1,
   STREAM_INPUT__DATA_BLOCK,
-  STREAM_INPUT__DATA_SCAN,
+  STREAM_INPUT__TABLE_SCAN,
+  STREAM_INPUT__TQ_SCAN,
   STREAM_INPUT__DATA_RETRIEVE,
   STREAM_INPUT__TRIGGER,
   STREAM_INPUT__CHECKPOINT,
@@ -122,7 +123,8 @@ enum {
 };
 
 typedef struct {
-  int8_t fetchType;
+  int8_t       fetchType;
+  STqOffsetVal offset;
   union {
     SSDataBlock data;
     void*       meta;

include/common/tdatablock.h

@@ -231,7 +231,7 @@ SSDataBlock* createDataBlock();
 int32_t      blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData);
 
 SColumnInfoData  createColumnInfoData(int16_t type, int32_t bytes, int16_t colId);
-SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index);
+SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index);
 
 void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress);
 const char* blockDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData);

include/common/tmsg.h

@@ -169,6 +169,9 @@ typedef enum _mgmt_table {
 #define TD_CHILD_TABLE  TSDB_CHILD_TABLE
 #define TD_NORMAL_TABLE TSDB_NORMAL_TABLE
 
+#define TD_REQ_FROM_APP  0
+#define TD_REQ_FROM_TAOX 1
+
 typedef struct {
   int32_t vgId;
   char*   dbFName;
@@ -432,25 +435,30 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapperEx(SDecoder* pDecoder, SSchemaW
 STSchema* tdGetSTSChemaFromSSChema(SSchema** pSchema, int32_t nCols);
 
 typedef struct {
-  char    name[TSDB_TABLE_FNAME_LEN];
-  int8_t  igExists;
-  int64_t delay1;
-  int64_t delay2;
-  int64_t watermark1;
-  int64_t watermark2;
-  int32_t ttl;
-  int32_t numOfColumns;
-  int32_t numOfTags;
-  int32_t numOfFuncs;
-  int32_t commentLen;
-  int32_t ast1Len;
-  int32_t ast2Len;
-  SArray* pColumns;  // array of SField
-  SArray* pTags;     // array of SField
-  SArray* pFuncs;
-  char*   pComment;
-  char*   pAst1;
-  char*   pAst2;
+  char     name[TSDB_TABLE_FNAME_LEN];
+  int8_t   igExists;
+  int8_t   source;  // 1-taosX or 0-taosClient
+  int8_t   reserved[6];
+  tb_uid_t suid;
+  int64_t  delay1;
+  int64_t  delay2;
+  int64_t  watermark1;
+  int64_t  watermark2;
+  int32_t  ttl;
+  int32_t  colVer;
+  int32_t  tagVer;
+  int32_t  numOfColumns;
+  int32_t  numOfTags;
+  int32_t  numOfFuncs;
+  int32_t  commentLen;
+  int32_t  ast1Len;
+  int32_t  ast2Len;
+  SArray*  pColumns;  // array of SField
+  SArray*  pTags;     // array of SField
+  SArray*  pFuncs;
+  char*    pComment;
+  char*    pAst1;
+  char*    pAst2;
 } SMCreateStbReq;
 
 int32_t tSerializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pReq);
@@ -458,8 +466,11 @@ int32_t tDeserializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pR
 void    tFreeSMCreateStbReq(SMCreateStbReq* pReq);
 
 typedef struct {
-  char   name[TSDB_TABLE_FNAME_LEN];
-  int8_t igNotExists;
+  char     name[TSDB_TABLE_FNAME_LEN];
+  int8_t   igNotExists;
+  int8_t   source;  // 1-taosX or 0-taosClient
+  int8_t   reserved[6];
+  tb_uid_t suid;
 } SMDropStbReq;
 
 int32_t tSerializeSMDropStbReq(void* buf, int32_t bufLen, SMDropStbReq* pReq);
@@ -468,8 +479,6 @@ int32_t tDeserializeSMDropStbReq(void* buf, int32_t bufLen, SMDropStbReq* pReq);
 typedef struct {
   char    name[TSDB_TABLE_FNAME_LEN];
   int8_t  alterType;
-  int32_t tagVer;
-  int32_t colVer;
   int32_t numOfFields;
   SArray* pFields;
   int32_t ttl;

include/common/tname.h

@@ -37,6 +37,8 @@ typedef struct SName {
   char    tname[TSDB_TABLE_NAME_LEN];
 } SName;
 
+SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName);
+
 int32_t tNameExtractFullName(const SName* name, char* dst);
 
 int32_t tNameLen(const SName* name);

include/libs/executor/executor.h

@@ -174,7 +174,13 @@ int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t le
  */
 int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts);
 
-int32_t qStreamPrepareScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts);
+int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts);
+
+int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset);
+
+int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset);
+
+void* qStreamExtractMetaMsg(qTaskInfo_t tinfo);
 
 void*   qExtractReaderFromStreamScanner(void* scanner);
 int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner);

include/libs/function/function.h

@@ -172,7 +172,13 @@ typedef struct tExprNode {
 
 void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *));
 
+typedef enum {
+  SHOULD_FREE_COLDATA    = 0x1,   // the newly created column data needs to be destroyed.
+  DELEGATED_MGMT_COLDATA = 0x2,   // input column data should not be released.
+} ECOLDATA_MGMT_TYPE_E;
+
 struct SScalarParam {
+  ECOLDATA_MGMT_TYPE_E type;
   SColumnInfoData *columnData;
   SHashObj        *pHashFilter;
   int32_t          hashValueType;

include/libs/parser/parser.h

@@ -96,6 +96,9 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols
                     char* tableName, char* msgBuf, int16_t msgBufLen);
 int32_t smlBuildOutput(void* handle, SHashObj* pVgHash);
 
+int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray);
+SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap);
+SArray* serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap);
 #ifdef __cplusplus
 }
 #endif

include/libs/transport/trpc.h

@@ -27,7 +27,7 @@ extern "C" {
 
 #define TAOS_CONN_SERVER 0
 #define TAOS_CONN_CLIENT 1
-#define IsReq(pMsg) (pMsg->msgType & 1U)
+#define IsReq(pMsg)      (pMsg->msgType & 1U)
 
 extern int32_t tsRpcHeadSize;
 

include/libs/wal/wal.h

@@ -24,42 +24,14 @@
 extern "C" {
 #endif
 
-#define wFatal(...)                                              \
-  {                                                              \
-    if (wDebugFlag & DEBUG_FATAL) {                              \
-      taosPrintLog("WAL FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \
-    }                                                            \
-  }
-#define wError(...)                                              \
-  {                                                              \
-    if (wDebugFlag & DEBUG_ERROR) {                              \
-      taosPrintLog("WAL ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
-    }                                                            \
-  }
-#define wWarn(...)                                             \
-  {                                                            \
-    if (wDebugFlag & DEBUG_WARN) {                             \
-      taosPrintLog("WAL WARN ", DEBUG_WARN, 255, __VA_ARGS__); \
-    }                                                          \
-  }
-#define wInfo(...)                                        \
-  {                                                       \
-    if (wDebugFlag & DEBUG_INFO) {                        \
-      taosPrintLog("WAL ", DEBUG_INFO, 255, __VA_ARGS__); \
-    }                                                     \
-  }
-#define wDebug(...)                                               \
-  {                                                               \
-    if (wDebugFlag & DEBUG_DEBUG) {                               \
-      taosPrintLog("WAL ", DEBUG_DEBUG, wDebugFlag, __VA_ARGS__); \
-    }                                                             \
-  }
-#define wTrace(...)                                               \
-  {                                                               \
-    if (wDebugFlag & DEBUG_TRACE) {                               \
-      taosPrintLog("WAL ", DEBUG_TRACE, wDebugFlag, __VA_ARGS__); \
-    }                                                             \
-  }
+// clang-format off
+#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", DEBUG_FATAL, 255,        __VA_ARGS__); }}
+#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", DEBUG_ERROR, 255,        __VA_ARGS__); }}
+#define wWarn(...)  { if (wDebugFlag & DEBUG_WARN)  { taosPrintLog("WAL WARN ",  DEBUG_WARN, 255,         __VA_ARGS__); }}
+#define wInfo(...)  { if (wDebugFlag & DEBUG_INFO)  { taosPrintLog("WAL ",       DEBUG_INFO, 255,         __VA_ARGS__); }}
+#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ",       DEBUG_DEBUG, wDebugFlag, __VA_ARGS__); }}
+#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ",       DEBUG_TRACE, wDebugFlag, __VA_ARGS__); }}
+// clang-format on
 
 #define WAL_PROTO_VER    0
 #define WAL_NOSUFFIX_LEN 20
@@ -161,6 +133,7 @@ typedef struct {
   int64_t        curFileFirstVer;
   int64_t        curVersion;
   int64_t        capacity;
+  int8_t         curInvalid;
   TdThreadMutex  mutex;
   SWalFilterCond cond;
   SWalCkHead    *pHead;
@@ -194,6 +167,7 @@ int32_t walRestoreFromSnapshot(SWal *, int64_t ver);
 SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond);
 void        walCloseReader(SWalReader *pRead);
 int32_t     walReadVer(SWalReader *pRead, int64_t ver);
+int32_t     walReadSeekVer(SWalReader *pRead, int64_t ver);
 int32_t     walNextValidMsg(SWalReader *pRead);
 
 // only for tq usage

include/util/taoserror.h

@@ -247,9 +247,11 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_COLUMN_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03AE)
 #define TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03AF)
 #define TSDB_CODE_MND_SINGLE_STB_MODE_DB        TAOS_DEF_ERROR_CODE(0, 0x03B0)
+#define TSDB_CODE_MND_INVALID_SCHEMA_VER        TAOS_DEF_ERROR_CODE(0, 0x03B1)
+#define TSDB_CODE_MND_STABLE_UID_NOT_MATCH      TAOS_DEF_ERROR_CODE(0, 0x03B2)
 
 // mnode-infoSchema
-#define TSDB_CODE_MND_INVALID_SYS_TABLENAME         TAOS_DEF_ERROR_CODE(0, 0x03BA)
+#define TSDB_CODE_MND_INVALID_SYS_TABLENAME     TAOS_DEF_ERROR_CODE(0, 0x03BA)
 
 // mnode-func
 #define TSDB_CODE_MND_FUNC_ALREADY_EXIST        TAOS_DEF_ERROR_CODE(0, 0x03C0)

include/util/tlog.h

@@ -94,7 +94,7 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons
 #define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }
 #define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); }
 // clang-format on
-
+#define BUF_PAGE_DEBUG
 #ifdef __cplusplus
 }
 #endif

packaging/tools/install.sh

@@ -30,6 +30,7 @@ configDir="/etc/taos"
 installDir="/usr/local/taos"
 adapterName="taosadapter"
 benchmarkName="taosBenchmark"
+tmqName="tmq_sim"
 dumpName="taosdump"
 demoName="taosdemo"
 
@@ -205,6 +206,7 @@ function install_bin() {
   [ -x ${install_main_dir}/bin/${adapterName} ] && ${csudo}ln -s ${install_main_dir}/bin/${adapterName} ${bin_link_dir}/${adapterName} || :
   [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || :
   [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || :
+  [ -x ${install_main_dir}/bin/${tmqName} ] && ${csudo}ln -s ${install_main_dir}/bin/${tmqName} ${bin_link_dir}/${tmqName} || :
   [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || :
   [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || :
   [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || :

packaging/tools/makepkg.sh

@@ -60,7 +60,7 @@ if [ "$pagMode" == "lite" ]; then
   strip ${build_dir}/bin/${serverName}
   strip ${build_dir}/bin/${clientName}
   # lite version doesn't include taosadapter,  which will lead to no restful interface
-  bin_files="${build_dir}/bin/${serverName} ${build_dir}/bin/${clientName} ${script_dir}/remove.sh ${script_dir}/startPre.sh ${build_dir}/bin/taosBenchmark"
+  bin_files="${build_dir}/bin/${serverName} ${build_dir}/bin/${clientName} ${script_dir}/remove.sh ${script_dir}/startPre.sh ${build_dir}/bin/taosBenchmark  ${build_dir}/bin/tmq_sim"
   taostools_bin_files=""
 else
 
@@ -78,6 +78,7 @@ else
 
   taostools_bin_files=" ${build_dir}/bin/taosdump \
       ${build_dir}/bin/taosBenchmark \
+      ${build_dir}/bin/tmq_sim \
       ${build_dir}/bin/TDinsight.sh \
       $tdinsight_caches"
 

source/client/src/clientHb.c

@@ -264,15 +264,12 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
 
 static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) {
   static int32_t emptyRspNum = 0;
-  if (code != 0) {
-    taosMemoryFreeClear(param);
-    return -1;
-  }
-
   char             *key = (char *)param;
   SClientHbBatchRsp pRsp = {0};
-  tDeserializeSClientHbBatchRsp(pMsg->pData, pMsg->len, &pRsp);
-
+  if (TSDB_CODE_SUCCESS == code) {
+    tDeserializeSClientHbBatchRsp(pMsg->pData, pMsg->len, &pRsp);
+  }
+  
   int32_t rspNum = taosArrayGetSize(pRsp.rsps);
 
   taosThreadMutexLock(&appInfo.mutex);
@@ -288,6 +285,10 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) {
 
   taosMemoryFreeClear(param);
 
+  if (code != 0) {
+    (*pInst)->onlineDnodes = 0;
+  }
+
   if (rspNum) {
     tscDebug("hb got %d rsp, %d empty rsp received before", rspNum,
              atomic_val_compare_exchange_32(&emptyRspNum, emptyRspNum, 0));

source/client/src/clientImpl.c

@@ -325,11 +325,13 @@ int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList) {
   if (pInfo->pQnodeList) {
     taosArrayDestroy(pInfo->pQnodeList);
     pInfo->pQnodeList = NULL;
+    tscDebug("QnodeList cleared in cluster 0x%" PRIx64, pInfo->clusterId);
   }
 
   if (pNodeList) {
     pInfo->pQnodeList = taosArrayDup(pNodeList);
     taosArraySort(pInfo->pQnodeList, compareQueryNodeLoad);
+    tscDebug("QnodeList updated in cluster 0x%" PRIx64 ", num:%d", pInfo->clusterId, taosArrayGetSize(pInfo->pQnodeList));
   }
   taosThreadMutexUnlock(&pInfo->qnodeMutex);
 

source/client/src/clientSml.c

@@ -1570,41 +1570,40 @@ static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsV
     return TSDB_CODE_TSC_INVALID_TIME_STAMP;
   }
 
+  *tsVal = timeDouble;
   size_t typeLen = strlen(type->valuestring);
   if (typeLen == 1 && (type->valuestring[0] == 's' || type->valuestring[0] == 'S')) {
     // seconds
-    timeDouble = timeDouble * 1e9;
+    *tsVal = *tsVal * NANOSECOND_PER_SEC;
+    timeDouble = timeDouble * NANOSECOND_PER_SEC;
     if (smlDoubleToInt64OverFlow(timeDouble)) {
       smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
       return TSDB_CODE_TSC_INVALID_TIME_STAMP;
     }
-    *tsVal = timeDouble;
   } else if (typeLen == 2 && (type->valuestring[1] == 's' || type->valuestring[1] == 'S')) {
     switch (type->valuestring[0]) {
       case 'm':
       case 'M':
         // milliseconds
-        timeDouble = timeDouble * 1e6;
+        *tsVal = *tsVal * NANOSECOND_PER_MSEC;
+        timeDouble = timeDouble * NANOSECOND_PER_MSEC;
         if (smlDoubleToInt64OverFlow(timeDouble)) {
           smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
           return TSDB_CODE_TSC_INVALID_TIME_STAMP;
         }
-        *tsVal = timeDouble;
         break;
       case 'u':
       case 'U':
         // microseconds
-        timeDouble = timeDouble * 1e3;
+        *tsVal = *tsVal * NANOSECOND_PER_USEC;
+        timeDouble = timeDouble * NANOSECOND_PER_USEC;
         if (smlDoubleToInt64OverFlow(timeDouble)) {
           smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
           return TSDB_CODE_TSC_INVALID_TIME_STAMP;
         }
-        *tsVal = timeDouble;
         break;
       case 'n':
       case 'N':
-        // nanoseconds
-        *tsVal = timeDouble;
         break;
       default:
         return TSDB_CODE_TSC_INVALID_JSON;
@@ -1641,21 +1640,23 @@ static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) {
     if (timeDouble < 0) {
       return TSDB_CODE_TSC_INVALID_TIME_STAMP;
     }
+
     uint8_t tsLen = smlGetTimestampLen((int64_t)timeDouble);
+    tsVal = (int64_t)timeDouble;
     if (tsLen == TSDB_TIME_PRECISION_SEC_DIGITS) {
-      timeDouble = timeDouble * 1e9;
+      tsVal = tsVal * NANOSECOND_PER_SEC;
+      timeDouble = timeDouble * NANOSECOND_PER_SEC;
       if (smlDoubleToInt64OverFlow(timeDouble)) {
         smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
         return TSDB_CODE_TSC_INVALID_TIME_STAMP;
       }
-      tsVal = timeDouble;
     } else if (tsLen == TSDB_TIME_PRECISION_MILLI_DIGITS) {
-      timeDouble = timeDouble * 1e6;
+      tsVal = tsVal * NANOSECOND_PER_MSEC;
+      timeDouble = timeDouble * NANOSECOND_PER_MSEC;
       if (smlDoubleToInt64OverFlow(timeDouble)) {
         smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
         return TSDB_CODE_TSC_INVALID_TIME_STAMP;
       }
-      tsVal = timeDouble;
     } else if (timeDouble == 0) {
       tsVal = taosGetTimestampNs();
     } else {

source/client/test/smlTest.cpp

@@ -1284,4 +1284,210 @@ TEST(testCase, sml_dup_time_Test) {
   ASSERT_EQ(taos_errno(pRes), 0);
   taos_free_result(pRes);
 }
-*/
+
+
+TEST(testCase, sml_16960_Test) {
+TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
+ASSERT_NE(taos, nullptr);
+
+TAOS_RES* pRes = taos_query(taos, "create database if not exists d16368 schemaless 1");
+taos_free_result(pRes);
+
+pRes = taos_query(taos, "use d16368");
+taos_free_result(pRes);
+
+const char *sql[] = {
+    "[\n"
+    "{\n"
+    "\"timestamp\":\n"
+    "\n"
+    "{ \"value\": 1349020800000, \"type\": \"ms\" }\n"
+    ",\n"
+    "\"value\":\n"
+    "\n"
+    "{ \"value\": 830525384, \"type\": \"int\" }\n"
+    ",\n"
+    "\"tags\": {\n"
+    "\"id\": \"stb00_0\",\n"
+    "\"t0\":\n"
+    "\n"
+    "{ \"value\": 83972721, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t1\":\n"
+    "\n"
+    "{ \"value\": 539147525, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t2\":\n"
+    "\n"
+    "{ \"value\": 618258572, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t3\":\n"
+    "\n"
+    "{ \"value\": -10536201, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t4\":\n"
+    "\n"
+    "{ \"value\": 349227409, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t5\":\n"
+    "\n"
+    "{ \"value\": 249347042, \"type\": \"int\" }\n"
+    "},\n"
+    "\"metric\": \"stb0\"\n"
+    "},\n"
+    "{\n"
+    "\"timestamp\":\n"
+    "\n"
+    "{ \"value\": 1349020800001, \"type\": \"ms\" }\n"
+    ",\n"
+    "\"value\":\n"
+    "\n"
+    "{ \"value\": -588348364, \"type\": \"int\" }\n"
+    ",\n"
+    "\"tags\": {\n"
+    "\"id\": \"stb00_0\",\n"
+    "\"t0\":\n"
+    "\n"
+    "{ \"value\": 83972721, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t1\":\n"
+    "\n"
+    "{ \"value\": 539147525, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t2\":\n"
+    "\n"
+    "{ \"value\": 618258572, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t3\":\n"
+    "\n"
+    "{ \"value\": -10536201, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t4\":\n"
+    "\n"
+    "{ \"value\": 349227409, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t5\":\n"
+    "\n"
+    "{ \"value\": 249347042, \"type\": \"int\" }\n"
+    "},\n"
+    "\"metric\": \"stb0\"\n"
+    "},\n"
+    "{\n"
+    "\"timestamp\":\n"
+    "\n"
+    "{ \"value\": 1349020800002, \"type\": \"ms\" }\n"
+    ",\n"
+    "\"value\":\n"
+    "\n"
+    "{ \"value\": -370310823, \"type\": \"int\" }\n"
+    ",\n"
+    "\"tags\": {\n"
+    "\"id\": \"stb00_0\",\n"
+    "\"t0\":\n"
+    "\n"
+    "{ \"value\": 83972721, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t1\":\n"
+    "\n"
+    "{ \"value\": 539147525, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t2\":\n"
+    "\n"
+    "{ \"value\": 618258572, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t3\":\n"
+    "\n"
+    "{ \"value\": -10536201, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t4\":\n"
+    "\n"
+    "{ \"value\": 349227409, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t5\":\n"
+    "\n"
+    "{ \"value\": 249347042, \"type\": \"int\" }\n"
+    "},\n"
+    "\"metric\": \"stb0\"\n"
+    "},\n"
+    "{\n"
+    "\"timestamp\":\n"
+    "\n"
+    "{ \"value\": 1349020800003, \"type\": \"ms\" }\n"
+    ",\n"
+    "\"value\":\n"
+    "\n"
+    "{ \"value\": -811250191, \"type\": \"int\" }\n"
+    ",\n"
+    "\"tags\": {\n"
+    "\"id\": \"stb00_0\",\n"
+    "\"t0\":\n"
+    "\n"
+    "{ \"value\": 83972721, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t1\":\n"
+    "\n"
+    "{ \"value\": 539147525, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t2\":\n"
+    "\n"
+    "{ \"value\": 618258572, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t3\":\n"
+    "\n"
+    "{ \"value\": -10536201, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t4\":\n"
+    "\n"
+    "{ \"value\": 349227409, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t5\":\n"
+    "\n"
+    "{ \"value\": 249347042, \"type\": \"int\" }\n"
+    "},\n"
+    "\"metric\": \"stb0\"\n"
+    "},\n"
+    "{\n"
+    "\"timestamp\":\n"
+    "\n"
+    "{ \"value\": 1349020800004, \"type\": \"ms\" }\n"
+    ",\n"
+    "\"value\":\n"
+    "\n"
+    "{ \"value\": -330340558, \"type\": \"int\" }\n"
+    ",\n"
+    "\"tags\": {\n"
+    "\"id\": \"stb00_0\",\n"
+    "\"t0\":\n"
+    "\n"
+    "{ \"value\": 83972721, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t1\":\n"
+    "\n"
+    "{ \"value\": 539147525, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t2\":\n"
+    "\n"
+    "{ \"value\": 618258572, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t3\":\n"
+    "\n"
+    "{ \"value\": -10536201, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t4\":\n"
+    "\n"
+    "{ \"value\": 349227409, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t5\":\n"
+    "\n"
+    "{ \"value\": 249347042, \"type\": \"int\" }\n"
+    "},\n"
+    "\"metric\": \"stb0\"\n"
+    "}\n"
+    "]"
+};
+
+pRes = taos_schemaless_insert(taos, (char**)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS);
+ASSERT_EQ(taos_errno(pRes), 0);
+taos_free_result(pRes);
+}
+*/
\ No newline at end of file

source/common/src/tdatablock.c

@@ -1356,7 +1356,7 @@ SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId)
   return col;
 }
 
-SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index) {
+SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index) {
   ASSERT(pBlock != NULL);
   if (index >= taosArrayGetSize(pBlock->pDataBlock)) {
     return NULL;
@@ -1735,9 +1735,9 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf)
   int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock);
   int32_t rows = pDataBlock->info.rows;
   int32_t len = 0;
-  len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|group id %lu|\n", flag,
+  len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|group id:%" PRIu64 "|\n", flag,
                   (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId);
-  if (len >= size -1) return dumpBuf;
+  if (len >= size - 1) return dumpBuf;
 
   for (int32_t j = 0; j < rows; j++) {
     len += snprintf(dumpBuf + len, size - len, "%s |", flag);

source/common/src/tmsg.c

@@ -381,9 +381,11 @@ static int32_t tDeserializeSClientHbRsp(SDecoder *pDecoder, SClientHbRsp *pRsp)
     if (pQnodeNum > 0) {
       pRsp->query->pQnodeList = taosArrayInit(pQnodeNum, sizeof(SQueryNodeLoad));
       if (NULL == pRsp->query->pQnodeList) return -1;
-      SQueryNodeLoad load = {0};
-      if (tDecodeSQueryNodeLoad(pDecoder, &load) < 0) return -1;
-      taosArrayPush(pRsp->query->pQnodeList, &load);
+      for (int32_t i = 0; i < pQnodeNum; ++i) {
+        SQueryNodeLoad load = {0};
+        if (tDecodeSQueryNodeLoad(pDecoder, &load) < 0) return -1;
+        taosArrayPush(pRsp->query->pQnodeList, &load);
+      }
     }
   }
 
@@ -496,11 +498,18 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->source) < 0) return -1;
+  for (int32_t i = 0; i < sizeof(pReq->reserved) / sizeof(int8_t); ++i) {
+    if (tEncodeI8(&encoder, pReq->reserved[i]) < 0) return -1;
+  }
+  if (tEncodeI64(&encoder, pReq->suid) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->delay1) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->delay2) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->watermark1) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->watermark2) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->colVer) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->tagVer) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->numOfColumns) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->numOfTags) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->numOfFuncs) < 0) return -1;
@@ -553,11 +562,18 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->source) < 0) return -1;
+  for (int32_t i = 0; i < sizeof(pReq->reserved) / sizeof(int8_t); ++i) {
+    if (tDecodeI8(&decoder, &pReq->reserved[i]) < 0) return -1;
+  }
+  if (tDecodeI64(&decoder, &pReq->suid) < 0) return -1;
   if (tDecodeI64(&decoder, &pReq->delay1) < 0) return -1;
   if (tDecodeI64(&decoder, &pReq->delay2) < 0) return -1;
   if (tDecodeI64(&decoder, &pReq->watermark1) < 0) return -1;
   if (tDecodeI64(&decoder, &pReq->watermark2) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->colVer) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->tagVer) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->numOfColumns) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->numOfTags) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->numOfFuncs) < 0) return -1;
@@ -645,6 +661,11 @@ int32_t tSerializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) {
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->source) < 0) return -1;
+  for (int32_t i = 0; i < sizeof(pReq->reserved) / sizeof(int8_t); ++i) {
+    if (tEncodeI8(&encoder, pReq->reserved[i]) < 0) return -1;
+  }
+  if (tEncodeI64(&encoder, pReq->suid) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -659,6 +680,12 @@ int32_t tDeserializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq)
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->source) < 0) return -1;
+  for (int32_t i = 0; i < sizeof(pReq->reserved) / sizeof(int8_t); ++i) {
+    if (tDecodeI8(&decoder, &pReq->reserved[i]) < 0) return -1;
+  }
+  if (tDecodeI64(&decoder, &pReq->suid) < 0) return -1;
+
   tEndDecode(&decoder);
 
   tDecoderClear(&decoder);
@@ -672,8 +699,6 @@ int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq)
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->alterType) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->tagVer) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->colVer) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->numOfFields) < 0) return -1;
   for (int32_t i = 0; i < pReq->numOfFields; ++i) {
     SField *pField = taosArrayGet(pReq->pFields, i);
@@ -700,8 +725,6 @@ int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->alterType) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->tagVer) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->colVer) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->numOfFields) < 0) return -1;
   pReq->pFields = taosArrayInit(pReq->numOfFields, sizeof(SField));
   if (pReq->pFields == NULL) {
@@ -5406,11 +5429,11 @@ int32_t tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) {
   } else if (pVal->type == TMQ_OFFSET__RESET_LATEST) {
     snprintf(buf, maxLen, "offset(reset to latest)");
   } else if (pVal->type == TMQ_OFFSET__LOG) {
-    snprintf(buf, maxLen, "offset(log) ver:%ld", pVal->version);
+    snprintf(buf, maxLen, "offset(log) ver:%" PRId64, pVal->version);
   } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_DATA) {
-    snprintf(buf, maxLen, "offset(ss data) uid:%ld, ts:%ld", pVal->uid, pVal->ts);
+    snprintf(buf, maxLen, "offset(ss data) uid:%" PRId64 ", ts:%" PRId64, pVal->uid, pVal->ts);
   } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_META) {
-    snprintf(buf, maxLen, "offset(ss meta) uid:%ld, ts:%ld", pVal->uid, pVal->ts);
+    snprintf(buf, maxLen, "offset(ss meta) uid:%" PRId64 ", ts:%" PRId64, pVal->uid, pVal->ts);
   } else {
     ASSERT(0);
   }

source/common/src/tname.c

@@ -115,6 +115,14 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in
 
 #endif
 
+SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName) {
+  pName->type = TSDB_TABLE_NAME_T;
+  pName->acctId = acctId;
+  strcpy(pName->dbname, pDbName);
+  strcpy(pName->tname, pTableName);
+  return pName;
+}
+
 int32_t tNameExtractFullName(const SName* name, char* dst) {
   assert(name != NULL && dst != NULL);
 

source/dnode/mgmt/mgmt_dnode/src/dmHandle.c

@@ -21,7 +21,7 @@ extern SConfig *tsCfg;
 
 static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) {
   if (pMgmt->pData->dnodeId == 0 || pMgmt->pData->clusterId == 0) {
-    dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId);
+    dInfo("set local info, dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId);
     taosThreadRwlockWrlock(&pMgmt->pData->lock);
     pMgmt->pData->dnodeId = pCfg->dnodeId;
     pMgmt->pData->clusterId = pCfg->clusterId;

source/dnode/mgmt/mgmt_mnode/src/mmWorker.c

@@ -167,7 +167,7 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
   if (pMsg == NULL) return -1;
   memcpy(pMsg, pRpc, sizeof(SRpcMsg));
 
-  dTrace("msg:%p, is created and will put int %s queue", pMsg, pWorker->name);
+  dTrace("msg:%p, is created and will put into %s queue, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType));
   return mmPutMsgToWorker(pMgmt, pWorker, pMsg);
 }
 

source/dnode/mgmt/node_mgmt/src/dmNodes.c

@@ -277,7 +277,7 @@ int32_t dmRunDnode(SDnode *pDnode) {
 
   while (1) {
     if (pDnode->stop) {
-      dInfo("dnode is about to stop");
+      dInfo("TDengine is about to stop");
       dmSetStatus(pDnode, DND_STAT_STOPPED);
       dmStopNodes(pDnode);
       dmCloseNodes(pDnode);

source/dnode/mnode/impl/inc/mndInt.h

@@ -34,8 +34,6 @@ extern "C" {
 #endif
 
 // clang-format off
-
-
 #define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", DEBUG_FATAL, 255,        __VA_ARGS__); }}
 #define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", DEBUG_ERROR, 255,        __VA_ARGS__); }}
 #define mWarn(...)  { if (mDebugFlag & DEBUG_WARN)  { taosPrintLog("MND WARN ",  DEBUG_WARN,  255,        __VA_ARGS__); }}
@@ -49,7 +47,6 @@ extern "C" {
 #define mGInfo(param, ...)  { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mInfo (param ", gtid:%s", __VA_ARGS__, buf);}
 #define mGDebug(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mDebug(param ", gtid:%s", __VA_ARGS__, buf);}
 #define mGTrace(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mTrace(param ", gtid:%s", __VA_ARGS__, buf);}
-
 // clang-format on
 
 #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE)

source/dnode/mnode/impl/src/mndConsumer.c

@@ -92,7 +92,7 @@ static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) {
   SMqConsumerObj     *pConsumer = mndAcquireConsumer(pMnode, pLostMsg->consumerId);
   ASSERT(pConsumer);
 
-  mInfo("receive consumer lost msg, consumer id %ld, status %s", pLostMsg->consumerId,
+  mInfo("receive consumer lost msg, consumer id %" PRId64 ", status %s", pLostMsg->consumerId,
         mndConsumerStatusName(pConsumer->status));
 
   if (pConsumer->status != MQ_CONSUMER_STATUS__READY) {
@@ -124,7 +124,7 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) {
   SMqConsumerObj        *pConsumer = mndAcquireConsumer(pMnode, pRecoverMsg->consumerId);
   ASSERT(pConsumer);
 
-  mInfo("receive consumer recover msg, consumer id %ld, status %s", pRecoverMsg->consumerId,
+  mInfo("receive consumer recover msg, consumer id %" PRId64 ", status %s", pRecoverMsg->consumerId,
         mndConsumerStatusName(pConsumer->status));
 
   if (pConsumer->status != MQ_CONSUMER_STATUS__READY) {
@@ -296,7 +296,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
   // 2. check epoch, only send ep info when epoches do not match
   if (epoch != serverEpoch) {
     taosRLockLatch(&pConsumer->lock);
-    mInfo("process ask ep, consumer %ld(epoch %d), server epoch %d", consumerId, epoch, serverEpoch);
+    mInfo("process ask ep, consumer:%" PRId64 "(epoch %d), server epoch %d", consumerId, epoch, serverEpoch);
     int32_t numOfTopics = taosArrayGetSize(pConsumer->currentTopics);
 
     rsp.topics = taosArrayInit(numOfTopics, sizeof(SMqSubTopicEp));
@@ -441,7 +441,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
     SMqTopicObj topicObj = {0};
     memcpy(&topicObj, pTopic, sizeof(SMqTopicObj));
     topicObj.refConsumerCnt = pTopic->refConsumerCnt + 1;
-    mInfo("subscribe topic %s by consumer %ld cgroup %s, refcnt %d", pTopic->name, consumerId, cgroup,
+    mInfo("subscribe topic %s by consumer:%" PRId64 ",cgroup %s, refcnt %d", pTopic->name, consumerId, cgroup,
           topicObj.refConsumerCnt);
     if (mndSetTopicCommitLogs(pMnode, pTrans, &topicObj) != 0) goto SUBSCRIBE_OVER;
 #endif
@@ -451,7 +451,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
 
   pConsumerOld = mndAcquireConsumer(pMnode, consumerId);
   if (pConsumerOld == NULL) {
-    mInfo("receive subscribe request from new consumer: %ld", consumerId);
+    mInfo("receive subscribe request from new consumer:%" PRId64, consumerId);
 
     pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup);
     tstrncpy(pConsumerNew->clientId, subscribe.clientId, 256);
@@ -472,7 +472,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
 
     int32_t status = atomic_load_32(&pConsumerOld->status);
 
-    mInfo("receive subscribe request from old consumer: %ld, current status: %s", consumerId,
+    mInfo("receive subscribe request from old consumer:%" PRId64 ", current status: %s", consumerId,
           mndConsumerStatusName(status));
 
     if (status != MQ_CONSUMER_STATUS__READY) {

source/dnode/mnode/impl/src/mndDnode.c

@@ -406,7 +406,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
     }
 
     if (statusReq.dnodeId == 0) {
-      mInfo("dnode:%d, %s first access, set clusterId %" PRId64, pDnode->id, pDnode->ep, pMnode->clusterId);
+      mInfo("dnode:%d, %s first access, clusterId:%" PRId64, pDnode->id, pDnode->ep, pMnode->clusterId);
     } else {
       if (statusReq.clusterId != pMnode->clusterId) {
         if (pDnode != NULL) {

source/dnode/mnode/impl/src/mndOffset.c

@@ -185,7 +185,7 @@ static int32_t mndProcessCommitOffsetReq(SRpcMsg *pMsg) {
 
   for (int32_t i = 0; i < commitOffsetReq.num; i++) {
     SMqOffset *pOffset = &commitOffsetReq.offsets[i];
-    mInfo("commit offset %ld to vg %d of consumer group %s on topic %s", pOffset->offset, pOffset->vgId,
+    mInfo("commit offset %" PRId64 " to vgId:%d of consumer group %s on topic %s", pOffset->offset, pOffset->vgId,
           pOffset->cgroup, pOffset->topicName);
     if (mndMakePartitionKey(key, pOffset->cgroup, pOffset->topicName, pOffset->vgId) < 0) {
       mError("submit offset to topic %s failed", pOffset->topicName);

source/dnode/mnode/impl/src/mndStb.c

@@ -45,6 +45,7 @@ static int32_t  mndProcessTableMetaReq(SRpcMsg *pReq);
 static int32_t  mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void     mndCancelGetNextStb(SMnode *pMnode, void *pIter);
 static int32_t  mndProcessTableCfgReq(SRpcMsg *pReq);
+static int32_t  mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp);
 
 int32_t mndInitStb(SMnode *pMnode) {
   SSdbTable table = {
@@ -705,10 +706,10 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
   memcpy(pDst->db, pDb->name, TSDB_DB_FNAME_LEN);
   pDst->createdTime = taosGetTimestampMs();
   pDst->updateTime = pDst->createdTime;
-  pDst->uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
+  pDst->uid = (pCreate->source == 1) ? pCreate->suid : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
   pDst->dbUid = pDb->uid;
-  pDst->tagVer = 1;
-  pDst->colVer = 1;
+  pDst->tagVer = (pCreate->source != TD_REQ_FROM_APP) ? pCreate->tagVer : 1;
+  pDst->colVer = (pCreate->source != TD_REQ_FROM_APP) ? pCreate->colVer : 1;
   pDst->smaVer = 1;
   pDst->nextColId = 1;
   pDst->maxdelay[0] = pCreate->delay1;
@@ -854,6 +855,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
   SStbObj       *pStb = NULL;
   SDbObj        *pDb = NULL;
   SMCreateStbReq createReq = {0};
+  bool           isAlter = false;
 
   if (tDeserializeSMCreateStbReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
@@ -869,9 +871,39 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
   pStb = mndAcquireStb(pMnode, createReq.name);
   if (pStb != NULL) {
     if (createReq.igExists) {
-      mDebug("stb:%s, already exist, ignore exist is set", createReq.name);
-      code = 0;
-      goto _OVER;
+      if (createReq.source == TD_REQ_FROM_APP) {
+        mDebug("stb:%s, already exist, ignore exist is set", createReq.name);
+        code = 0;
+        goto _OVER;
+      } else if (pStb->uid != createReq.suid) {
+        mError("stb:%s, already exist while create, input suid:%" PRId64 " not match with exist suid:%" PRId64,
+               createReq.name, createReq.suid, pStb->uid);
+        terrno = TSDB_CODE_MND_STABLE_UID_NOT_MATCH;
+        goto _OVER;
+      } else if (createReq.tagVer > 0 || createReq.colVer > 0) {
+        int32_t tagDelta = pStb->tagVer - createReq.tagVer;
+        int32_t colDelta = pStb->colVer - createReq.colVer;
+        int32_t verDelta = tagDelta + verDelta;
+        mInfo("stb:%s, already exist while create, input tagVer:%d colVer:%d, exist tagVer:%d colVer:%d",
+              createReq.name, createReq.tagVer, createReq.colVer, pStb->tagVer, pStb->colVer);
+        if (tagDelta <= 0 && colDelta <= 0) {
+          mInfo("stb:%s, schema version is not incremented and nothing needs to be done", createReq.name);
+          code = 0;
+          goto _OVER;
+        } else if ((tagDelta == 1 || colDelta == 1) && (verDelta == 1)) {
+          isAlter = true;
+          mInfo("stb:%s, schema version is only increased by 1 number, do alter operation", createReq.name);
+        } else {
+          mError("stb:%s, schema version increase more than 1 number, error is returned", createReq.name);
+          terrno = TSDB_CODE_MND_INVALID_SCHEMA_VER;
+          goto _OVER;
+        }
+      } else {
+        mError("stb:%s, already exist while create, input tagVer:%d colVer:%d is invalid", createReq.name,
+               createReq.tagVer, createReq.colVer, pStb->tagVer, pStb->colVer);
+        terrno = TSDB_CODE_MND_INVALID_SCHEMA_VER;
+        goto _OVER;
+      }
     } else {
       terrno = TSDB_CODE_MND_STB_ALREADY_EXIST;
       goto _OVER;
@@ -900,7 +932,12 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  code = mndCreateStb(pMnode, pReq, &createReq, pDb);
+  if (isAlter) {
+    bool needRsp = false;
+    code = mndAlterStbImp(pMnode, pReq, pDb, pStb, needRsp);
+  } else {
+    code = mndCreateStb(pMnode, pReq, &createReq, pDb);
+  }
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 
 _OVER:
@@ -1466,14 +1503,13 @@ static int32_t mndBuildStbCfg(SMnode *pMnode, const char *dbFName, const char *t
   return code;
 }
 
-static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, SStbObj *pObj, void **pCont,
-                                     int32_t *pLen) {
+static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, SStbObj *pObj, void **pCont, int32_t *pLen) {
   int32_t       ret;
   SEncoder      ec = {0};
   uint32_t      contLen = 0;
   SMAlterStbRsp alterRsp = {0};
   SName         name = {0};
-  tNameFromString(&name, pAlter->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
+  tNameFromString(&name, pObj->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
 
   alterRsp.pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
   if (NULL == alterRsp.pMeta) {
@@ -1506,10 +1542,36 @@ static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, S
   return 0;
 }
 
+static int32_t mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp) {
+  int32_t code = -1;
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq);
+  if (pTrans == NULL) goto _OVER;
+
+  mDebug("trans:%d, used to alter stb:%s", pTrans->id, pStb->name);
+  mndTransSetDbName(pTrans, pDb->name, NULL);
+
+  if (needRsp) {
+    void   *pCont = NULL;
+    int32_t contLen = 0;
+    if (mndBuildSMAlterStbRsp(pDb, pStb, &pCont, &contLen) != 0) goto _OVER;
+    mndTransSetRpcRsp(pTrans, pCont, contLen);
+  }
+
+  if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+  if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+  if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+  if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
+
+  code = 0;
+
+_OVER:
+  mndTransDrop(pTrans);
+  return code;
+}
+
 static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *pAlter, SDbObj *pDb, SStbObj *pOld) {
   bool    needRsp = true;
   int32_t code = -1;
-  STrans *pTrans = NULL;
   SField *pField0 = NULL;
 
   SStbObj stbObj = {0};
@@ -1558,30 +1620,9 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p
   }
 
   if (code != 0) goto _OVER;
-
-  code = -1;
-  pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq);
-  if (pTrans == NULL) goto _OVER;
-
-  mDebug("trans:%d, used to alter stb:%s", pTrans->id, pAlter->name);
-  mndTransSetDbName(pTrans, pDb->name, NULL);
-
-  if (needRsp) {
-    void   *pCont = NULL;
-    int32_t contLen = 0;
-    if (mndBuildSMAlterStbRsp(pDb, pAlter, &stbObj, &pCont, &contLen) != 0) goto _OVER;
-    mndTransSetRpcRsp(pTrans, pCont, contLen);
-  }
-
-  if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER;
-  if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER;
-  if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER;
-  if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
-
-  code = 0;
+  code = mndAlterStbImp(pMnode, pReq, pDb, &stbObj, needRsp);
 
 _OVER:
-  mndTransDrop(pTrans);
   taosMemoryFreeClear(stbObj.pTags);
   taosMemoryFreeClear(stbObj.pColumns);
   return code;
@@ -1614,14 +1655,6 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if ((alterReq.tagVer > 0 && alterReq.colVer > 0) &&
-      (alterReq.tagVer <= pStb->tagVer || alterReq.colVer <= pStb->colVer)) {
-    mDebug("stb:%s, already exist, tagVer:%d colVer:%d smaller than in mnode, tagVer:%d colVer:%d, alter success",
-           alterReq.name, alterReq.tagVer, alterReq.colVer, pStb->tagVer, pStb->colVer);
-    code = 0;
-    goto _OVER;
-  }
-
   if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
     goto _OVER;
   }
@@ -1752,6 +1785,11 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) {
     }
   }
 
+  if (dropReq.source != TD_REQ_FROM_APP && pStb->uid != dropReq.suid) {
+    terrno = TSDB_CODE_MND_STB_NOT_EXIST;
+    goto _OVER;
+  }
+
   pDb = mndAcquireDbByStb(pMnode, dropReq.name);
   if (pDb == NULL) {
     terrno = TSDB_CODE_MND_DB_NOT_SELECTED;

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -235,7 +235,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
             .pVgEp = pVgEp,
         };
         taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
-        mInfo("mq rebalance: remove vg %d from consumer %ld", pVgEp->vgId, consumerId);
+        mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64, pVgEp->vgId, consumerId);
       }
       taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
       // put into removed
@@ -255,7 +255,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
           .pVgEp = pVgEp,
       };
       taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg));
-      mInfo("mq rebalance: remove vg %d from unassigned", pVgEp->vgId);
+      mInfo("mq rebalance: remove vgId:%d from unassigned", pVgEp->vgId);
     }
   }
 
@@ -298,7 +298,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
                 .pVgEp = pVgEp,
             };
             taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
-            mInfo("mq rebalance: remove vg %d from consumer %ld (first scan)", pVgEp->vgId, pConsumerEp->consumerId);
+            mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, pConsumerEp->consumerId);
           }
           imbCnt++;
         }
@@ -312,7 +312,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
               .pVgEp = pVgEp,
           };
           taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
-          mInfo("mq rebalance: remove vg %d from consumer %ld (first scan)", pVgEp->vgId, pConsumerEp->consumerId);
+          mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, pConsumerEp->consumerId);
         }
       }
     }
@@ -329,7 +329,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
       newConsumerEp.vgs = taosArrayInit(0, sizeof(void *));
       taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp));
       taosArrayPush(pOutput->newConsumers, &consumerId);
-      mInfo("mq rebalance: add new consumer %ld", consumerId);
+      mInfo("mq rebalance: add new consumer:%" PRId64, consumerId);
     }
   }
 
@@ -354,7 +354,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
       taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
       pRebVg->newConsumerId = pConsumerEp->consumerId;
       taosArrayPush(pOutput->rebVgs, pRebVg);
-      mInfo("mq rebalance: add vg %d to consumer %ld (second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId);
+      mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId);
     }
   }
 
@@ -372,7 +372,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
       taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
       pRebVg->newConsumerId = pConsumerEp->consumerId;
       taosArrayPush(pOutput->rebVgs, pRebVg);
-      mInfo("mq rebalance: add vg %d to consumer %ld (second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId);
+      mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId);
     }
   } else {
     // if all consumer is removed, put all vg into unassigned
@@ -385,7 +385,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
       ASSERT(pRebOutput->newConsumerId == -1);
       taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp);
       taosArrayPush(pOutput->rebVgs, pRebOutput);
-      mInfo("mq rebalance: unassign vg %d (second scan)", pRebOutput->pVgEp->vgId);
+      mInfo("mq rebalance: unassign vgId:%d (second scan)", pRebOutput->pVgEp->vgId);
     }
   }
 
@@ -393,7 +393,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
   mInfo("rebalance calculation completed, rebalanced vg:");
   for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) {
     SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i);
-    mInfo("vgId:%d, moved from consumer %" PRId64 " to consumer %" PRId64, pOutputRebVg->pVgEp->vgId,
+    mInfo("vgId:%d, moved from consumer:%" PRId64 ", to consumer:%" PRId64, pOutputRebVg->pVgEp->vgId,
           pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId);
   }
 
@@ -546,7 +546,11 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
       char cgroup[TSDB_CGROUP_LEN];
       mndSplitSubscribeKey(pRebInfo->key, topic, cgroup, true);
       SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
-      ASSERT(pTopic);
+      /*ASSERT(pTopic);*/
+      if (pTopic == NULL) {
+        mError("rebalance %s failed since topic %s was dropped, abort", pRebInfo->key, topic);
+        continue;
+      }
       taosRLockLatch(&pTopic->lock);
 
       rebOutput.pSub = mndCreateSub(pMnode, pTopic, pRebInfo->key);

source/dnode/mnode/impl/src/mndTopic.c

@@ -558,7 +558,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
         mndReleaseConsumer(pMnode, pConsumer);
         mndReleaseTopic(pMnode, pTopic);
         terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED;
-        mError("topic:%s, failed to drop since subscribed by consumer %ld in consumer group %s", dropReq.name,
+        mError("topic:%s, failed to drop since subscribed by consumer:%" PRId64 ", in consumer group %s", dropReq.name,
                pConsumer->consumerId, pConsumer->cgroup);
         return -1;
       }

source/dnode/mnode/impl/test/stb/stb.cpp

@@ -277,8 +277,6 @@ void* MndTestStb::BuildAlterStbUpdateColumnBytesReq(const char* stbname, const c
   req.numOfFields = 1;
   req.pFields = taosArrayInit(1, sizeof(SField));
   req.alterType = TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES;
-  req.tagVer = verInBlock;
-  req.colVer = verInBlock;
 
   SField field = {0};
   field.bytes = bytes;
@@ -818,7 +816,7 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) {
   {
     void*    pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col_not_exist", 20, &contLen, 1);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
-    ASSERT_EQ(pRsp->code, 0);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_NOT_EXIST);
 
     test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname);
     EXPECT_EQ(test.GetShowRows(), 1);

source/dnode/snode/src/snode.c

@@ -111,7 +111,7 @@ static int32_t sndProcessTaskDeployReq(SSnode *pNode, SRpcMsg *pMsg) {
 
   streamSetupTrigger(pTask);
 
-  qInfo("deploy stream: stream id %ld task id %d child id %d on snode", pTask->streamId, pTask->taskId,
+  qInfo("deploy stream: stream id %" PRId64 " task id %d child id %d on snode", pTask->streamId, pTask->taskId,
         pTask->selfChildId);
 
   taosHashPut(pMeta->pHash, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void *));

source/dnode/vnode/CMakeLists.txt

@@ -3,6 +3,7 @@ add_library(vnode STATIC "")
 target_sources(
     vnode
     PRIVATE
+
     # vnode
     "src/vnd/vnodeOpen.c"
     "src/vnd/vnodeBufPool.c"
@@ -13,7 +14,6 @@ target_sources(
     "src/vnd/vnodeSvr.c"
     "src/vnd/vnodeSync.c"
     "src/vnd/vnodeSnapshot.c"
-    "src/vnd/vnodeUtil.c"
 
     # meta
     "src/meta/metaOpen.c"
@@ -26,7 +26,6 @@ target_sources(
     "src/meta/metaSnapshot.c"
 
     # sma
-    "src/sma/sma.c"
     "src/sma/smaEnv.c"
     "src/sma/smaUtil.c"
     "src/sma/smaOpen.c"
@@ -47,6 +46,7 @@ target_sources(
     "src/tsdb/tsdbUtil.c"
     "src/tsdb/tsdbSnapshot.c"
     "src/tsdb/tsdbCacheRead.c"
+    "src/tsdb/tsdbRetention.c"
 
     # tq
     "src/tq/tq.c"
@@ -63,7 +63,6 @@ target_include_directories(
     PUBLIC "inc"
     PRIVATE "src/inc"
     PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar"
-    
 )
 target_link_libraries(
     vnode
@@ -77,18 +76,19 @@ target_link_libraries(
     PUBLIC executor
     PUBLIC scheduler
     PUBLIC tdb
-    #PUBLIC bdb
-    #PUBLIC scalar
+
+    # PUBLIC bdb
+    # PUBLIC scalar
     PUBLIC transport
     PUBLIC stream
     PUBLIC index
 )
 target_compile_definitions(vnode PUBLIC -DMETA_REFACT)
-if (${BUILD_WITH_INVERTEDINDEX})
-  add_definitions(-DUSE_INVERTED_INDEX) 
+
+if(${BUILD_WITH_INVERTEDINDEX})
+    add_definitions(-DUSE_INVERTED_INDEX)
 endif(${BUILD_WITH_INVERTEDINDEX})
+
 if(${BUILD_TEST})
     add_subdirectory(test)
 endif(${BUILD_TEST})
-
-

source/dnode/vnode/inc/vnode.h

@@ -38,10 +38,11 @@ extern "C" {
 #endif
 
 // vnode
-typedef struct SVnode           SVnode;
-typedef struct STsdbCfg         STsdbCfg;  // todo: remove
-typedef struct SVnodeCfg        SVnodeCfg;
-typedef struct SVSnapshotReader SVSnapshotReader;
+typedef struct SVnode       SVnode;
+typedef struct STsdbCfg     STsdbCfg;  // todo: remove
+typedef struct SVnodeCfg    SVnodeCfg;
+typedef struct SVSnapReader SVSnapReader;
+typedef struct SVSnapWriter SVSnapWriter;
 
 extern const SVnodeCfg vnodeCfgDefault;
 
@@ -57,10 +58,6 @@ void    vnodeStop(SVnode *pVnode);
 int64_t vnodeGetSyncHandle(SVnode *pVnode);
 void    vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
 void    vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId);
-int32_t vnodeSnapshotReaderOpen(SVnode *pVnode, SVSnapshotReader **ppReader, int64_t sver, int64_t ever);
-int32_t vnodeSnapshotReaderClose(SVSnapshotReader *pReader);
-int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32_t *nData);
-
 int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen);
 int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list);
 int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list);
@@ -174,6 +171,9 @@ int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList);
 int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *tbUidList);
 int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList);
 
+int32_t tqSeekVer(STqReader *pReader, int64_t ver);
+int32_t tqNextBlock(STqReader *pReader, SFetchRet *ret);
+
 int32_t tqReaderSetDataMsg(STqReader *pReader, SSubmitReq *pMsg, int64_t ver);
 bool    tqNextDataBlock(STqReader *pReader);
 bool    tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
@@ -182,7 +182,14 @@ int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader);
 // sma
 int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
 
-// need to reposition
+// SVSnapReader
+int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapReader **ppReader);
+int32_t vnodeSnapReaderClose(SVSnapReader *pReader);
+int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData);
+// SVSnapWriter
+int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter);
+int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback);
+int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData);
 
 // structs
 struct STsdbCfg {

source/dnode/vnode/src/inc/meta.h

@@ -57,6 +57,9 @@ int  metaRemoveTableFromIdx(SMeta* pMeta, tb_uid_t uid);
 // metaCommit ==================
 static FORCE_INLINE tb_uid_t metaGenerateUid(SMeta* pMeta) { return tGenIdPI64(); }
 
+// metaTable ==================
+int metaHandleEntry(SMeta* pMeta, const SMetaEntry* pME);
+
 struct SMeta {
   TdThreadRwlock lock;
 

source/dnode/vnode/src/inc/sma.h

@@ -47,7 +47,9 @@ struct SSmaEnv {
 };
 
 typedef struct {
-  int32_t smaRef;
+  int8_t  inited;
+  int32_t rsetId;
+  void   *tmrHandle;  // shared by all fetch tasks
 } SSmaMgmt;
 
 #define SMA_ENV_LOCK(env) ((env)->lock)
@@ -64,7 +66,6 @@ struct SRSmaStat {
   SSma     *pSma;
   int64_t   submitVer;
   int64_t   refId;         // shared by fetch tasks
-  void     *tmrHandle;     // shared by fetch tasks
   int8_t    triggerStat;   // shared by fetch tasks
   int8_t    runningStat;   // for persistence task 
   SHashObj *rsmaInfoHash;  // key: stbUid, value: SRSmaInfo;
@@ -81,7 +82,6 @@ struct SSmaStat {
 #define SMA_TSMA_STAT(s)     (&(s)->tsmaStat)
 #define SMA_RSMA_STAT(s)     (&(s)->rsmaStat)
 #define RSMA_INFO_HASH(r)    ((r)->rsmaInfoHash)
-#define RSMA_TMR_HANDLE(r)   ((r)->tmrHandle)
 #define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat)
 #define RSMA_RUNNING_STAT(r) (&(r)->runningStat)
 #define RSMA_REF_ID(r)       ((r)->refId)
@@ -95,6 +95,7 @@ enum {
   TASK_TRIGGER_STAT_CANCELLED = 4,
   TASK_TRIGGER_STAT_FINISHED = 5,
 };
+
 void  tdDestroySmaEnv(SSmaEnv *pSmaEnv);
 void *tdFreeSmaEnv(SSmaEnv *pSmaEnv);
 
@@ -104,6 +105,10 @@ int32_t tdInsertRSmaData(SSma *pSma, char *msg);
 
 int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat);
 int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat);
+
+void   *tdAcquireSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln);
+int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln);
+
 int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType);
 
 int32_t tdLockSma(SSma *pSma);
@@ -183,7 +188,7 @@ static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) {
 
 static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType);
 void          *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType);
-void          *tdFreeRSmaInfo(SRSmaInfo *pInfo);
+void          *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo);
 int32_t        tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat);
 
 int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName);

source/dnode/vnode/src/inc/tq.h

@@ -129,6 +129,7 @@ typedef struct {
 static STqMgmt tqMgmt = {0};
 
 // tqRead
+int64_t tqScan(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* offset);
 int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum);
 
 // tqExec

source/dnode/vnode/src/inc/tsdb.h

@@ -64,6 +64,7 @@ typedef struct SRowIter       SRowIter;
 typedef struct STsdbFS        STsdbFS;
 typedef struct SRowMerger     SRowMerger;
 typedef struct STsdbFSState   STsdbFSState;
+typedef struct STsdbSnapHdr   STsdbSnapHdr;
 
 #define TSDB_MAX_SUBBLOCKS 8
 #define TSDB_FHDR_SIZE     512

source/dnode/vnode/src/inc/vnodeInt.h

@@ -49,17 +49,20 @@
 extern "C" {
 #endif
 
-typedef struct SVnodeInfo          SVnodeInfo;
-typedef struct SMeta               SMeta;
-typedef struct SSma                SSma;
-typedef struct STsdb               STsdb;
-typedef struct STQ                 STQ;
-typedef struct SVState             SVState;
-typedef struct SVBufPool           SVBufPool;
-typedef struct SQWorker            SQHandle;
-typedef struct STsdbKeepCfg        STsdbKeepCfg;
-typedef struct SMetaSnapshotReader SMetaSnapshotReader;
-typedef struct STsdbSnapshotReader STsdbSnapshotReader;
+typedef struct SVnodeInfo      SVnodeInfo;
+typedef struct SMeta           SMeta;
+typedef struct SSma            SSma;
+typedef struct STsdb           STsdb;
+typedef struct STQ             STQ;
+typedef struct SVState         SVState;
+typedef struct SVBufPool       SVBufPool;
+typedef struct SQWorker        SQHandle;
+typedef struct STsdbKeepCfg    STsdbKeepCfg;
+typedef struct SMetaSnapReader SMetaSnapReader;
+typedef struct SMetaSnapWriter SMetaSnapWriter;
+typedef struct STsdbSnapReader STsdbSnapReader;
+typedef struct STsdbSnapWriter STsdbSnapWriter;
+typedef struct SSnapDataHdr    SSnapDataHdr;
 
 #define VNODE_META_DIR  "meta"
 #define VNODE_TSDB_DIR  "tsdb"
@@ -72,10 +75,8 @@ typedef struct STsdbSnapshotReader STsdbSnapshotReader;
 #define VNODE_RSMA2_DIR "rsma2"
 
 // vnd.h
-void*   vnodeBufPoolMalloc(SVBufPool* pPool, int size);
-void    vnodeBufPoolFree(SVBufPool* pPool, void* p);
-int32_t vnodeRealloc(void** pp, int32_t size);
-void    vnodeFree(void* p);
+void* vnodeBufPoolMalloc(SVBufPool* pPool, int size);
+void  vnodeBufPoolFree(SVBufPool* pPool, void* p);
 
 // meta
 typedef struct SMCtbCursor SMCtbCursor;
@@ -109,9 +110,6 @@ STSma*          metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid);
 STSmaWrapper*   metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid, bool deepCopy);
 SArray*         metaGetSmaIdsByTable(SMeta* pMeta, tb_uid_t uid);
 SArray*         metaGetSmaTbUids(SMeta* pMeta);
-int32_t         metaSnapshotReaderOpen(SMeta* pMeta, SMetaSnapshotReader** ppReader, int64_t sver, int64_t ever);
-int32_t         metaSnapshotReaderClose(SMetaSnapshotReader* pReader);
-int32_t         metaSnapshotRead(SMetaSnapshotReader* pReader, void** ppData, uint32_t* nData);
 void*           metaGetIdx(SMeta* pMeta);
 void*           metaGetIvtIdx(SMeta* pMeta);
 int             metaTtlSmaller(SMeta* pMeta, uint64_t time, SArray* uidList);
@@ -131,9 +129,6 @@ int32_t     tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* p
 int32_t     tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
 STsdbReader tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId,
                                 void* pMemRef);
-int32_t     tsdbSnapshotReaderOpen(STsdb* pTsdb, STsdbSnapshotReader** ppReader, int64_t sver, int64_t ever);
-int32_t     tsdbSnapshotReaderClose(STsdbSnapshotReader* pReader);
-int32_t     tsdbSnapshotRead(STsdbSnapshotReader* pReader, void** ppData, uint32_t* nData);
 
 // tq
 int     tqInit();
@@ -163,6 +158,8 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool
                             const char* stbFullName, int32_t vgId);
 
 // sma
+int32_t smaInit();
+void    smaCleanUp();
 int32_t smaOpen(SVnode* pVnode);
 int32_t smaClose(SSma* pSma);
 int32_t smaBegin(SSma* pSma);
@@ -181,6 +178,23 @@ int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore);
 void    tdUidStoreDestory(STbUidStore* pStore);
 void*   tdUidStoreFree(STbUidStore* pStore);
 
+// SMetaSnapReader ========================================
+int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader);
+int32_t metaSnapReaderClose(SMetaSnapReader** ppReader);
+int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData);
+// SMetaSnapWriter ========================================
+int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter);
+int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
+int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback);
+// STsdbSnapReader ========================================
+int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapReader** ppReader);
+int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader);
+int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData);
+// STsdbSnapWriter ========================================
+int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter);
+int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
+int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback);
+
 typedef struct {
   int8_t  streamType;  // sma or other
   int8_t  dstType;
@@ -200,7 +214,9 @@ typedef struct {
 struct SVState {
   int64_t committed;
   int64_t applied;
+  int64_t applyTerm;
   int64_t commitID;
+  int64_t commitTerm;
 };
 
 struct SVnodeInfo {
@@ -289,6 +305,12 @@ struct SSma {
 // sma
 void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);
 
+struct SSnapDataHdr {
+  int8_t  type;
+  int64_t size;
+  uint8_t data[];
+};
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/meta/metaSnapshot.c

@@ -15,53 +15,57 @@
 
 #include "meta.h"
 
-struct SMetaSnapshotReader {
+// SMetaSnapReader ========================================
+struct SMetaSnapReader {
   SMeta*  pMeta;
-  TBC*    pTbc;
   int64_t sver;
   int64_t ever;
+  TBC*    pTbc;
 };
 
-int32_t metaSnapshotReaderOpen(SMeta* pMeta, SMetaSnapshotReader** ppReader, int64_t sver, int64_t ever) {
-  int32_t              code = 0;
-  int32_t              c = 0;
-  SMetaSnapshotReader* pMetaReader = NULL;
+int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader) {
+  int32_t          code = 0;
+  int32_t          c = 0;
+  SMetaSnapReader* pMetaSnapReader = NULL;
 
-  pMetaReader = (SMetaSnapshotReader*)taosMemoryCalloc(1, sizeof(*pMetaReader));
-  if (pMetaReader == NULL) {
+  // alloc
+  pMetaSnapReader = (SMetaSnapReader*)taosMemoryCalloc(1, sizeof(*pMetaSnapReader));
+  if (pMetaSnapReader == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
   }
-  pMetaReader->pMeta = pMeta;
-  pMetaReader->sver = sver;
-  pMetaReader->ever = ever;
-  code = tdbTbcOpen(pMeta->pTbDb, &pMetaReader->pTbc, NULL);
+  pMetaSnapReader->pMeta = pMeta;
+  pMetaSnapReader->sver = sver;
+  pMetaSnapReader->ever = ever;
+
+  // impl
+  code = tdbTbcOpen(pMeta->pTbDb, &pMetaSnapReader->pTbc, NULL);
   if (code) {
     goto _err;
   }
 
-  code = tdbTbcMoveTo(pMetaReader->pTbc, &(STbDbKey){.version = sver, .uid = INT64_MIN}, sizeof(STbDbKey), &c);
+  code = tdbTbcMoveTo(pMetaSnapReader->pTbc, &(STbDbKey){.version = sver, .uid = INT64_MIN}, sizeof(STbDbKey), &c);
   if (code) {
     goto _err;
   }
 
-  *ppReader = pMetaReader;
+  *ppReader = pMetaSnapReader;
   return code;
 
 _err:
+  metaError("vgId:%d meta snap reader open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
   *ppReader = NULL;
   return code;
 }
 
-int32_t metaSnapshotReaderClose(SMetaSnapshotReader* pReader) {
-  if (pReader) {
-    tdbTbcClose(pReader->pTbc);
-    taosMemoryFree(pReader);
-  }
+int32_t metaSnapReaderClose(SMetaSnapReader** ppReader) {
+  tdbTbcClose((*ppReader)->pTbc);
+  taosMemoryFree(*ppReader);
+  *ppReader = NULL;
   return 0;
 }
 
-int32_t metaSnapshotRead(SMetaSnapshotReader* pReader, void** ppData, uint32_t* nDatap) {
+int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) {
   const void* pKey = NULL;
   const void* pData = NULL;
   int32_t     nKey = 0;
@@ -71,23 +75,110 @@ int32_t metaSnapshotRead(SMetaSnapshotReader* pReader, void** ppData, uint32_t*
   for (;;) {
     code = tdbTbcGet(pReader->pTbc, &pKey, &nKey, &pData, &nData);
     if (code || ((STbDbKey*)pData)->version > pReader->ever) {
-      return TSDB_CODE_VND_READ_END;
+      code = TSDB_CODE_VND_READ_END;
+      goto _exit;
     }
 
     if (((STbDbKey*)pData)->version < pReader->sver) {
+      tdbTbcMoveToNext(pReader->pTbc);
       continue;
     }
 
+    tdbTbcMoveToNext(pReader->pTbc);
     break;
   }
 
   // copy the data
-  if (vnodeRealloc(ppData, nData) < 0) {
+  if (tRealloc(ppData, sizeof(SSnapDataHdr) + nData) < 0) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     return code;
   }
+  ((SSnapDataHdr*)(*ppData))->type = 0;  // TODO: use macro
+  ((SSnapDataHdr*)(*ppData))->size = nData;
+  memcpy(((SSnapDataHdr*)(*ppData))->data, pData, nData);
+
+_exit:
+  return code;
+}
 
-  memcpy(*ppData, pData, nData);
-  *nDatap = nData;
+// SMetaSnapWriter ========================================
+struct SMetaSnapWriter {
+  SMeta*  pMeta;
+  int64_t sver;
+  int64_t ever;
+};
+
+static int32_t metaSnapRollback(SMetaSnapWriter* pWriter) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+static int32_t metaSnapCommit(SMetaSnapWriter* pWriter) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter) {
+  int32_t          code = 0;
+  SMetaSnapWriter* pWriter;
+
+  // alloc
+  pWriter = (SMetaSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter));
+  if (pWriter == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pWriter->pMeta = pMeta;
+  pWriter->sver = sver;
+  pWriter->ever = ever;
+
+  *ppWriter = pWriter;
+  return code;
+
+_err:
+  metaError("vgId:%d meta snapshot writer open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
+  *ppWriter = NULL;
+  return code;
+}
+
+int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback) {
+  int32_t          code = 0;
+  SMetaSnapWriter* pWriter = *ppWriter;
+
+  if (rollback) {
+    code = metaSnapRollback(pWriter);
+    if (code) goto _err;
+  } else {
+    code = metaSnapCommit(pWriter);
+    if (code) goto _err;
+  }
+  taosMemoryFree(pWriter);
+  *ppWriter = NULL;
+
+  return code;
+
+_err:
+  metaError("vgId:%d meta snapshot writer close failed since %s", TD_VID(pWriter->pMeta->pVnode), tstrerror(code));
+  return code;
+}
+
+int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
+  int32_t    code = 0;
+  SMeta*     pMeta = pWriter->pMeta;
+  SMetaEntry metaEntry = {0};
+  SDecoder*  pDecoder = &(SDecoder){0};
+
+  tDecoderInit(pDecoder, pData, nData);
+  metaDecodeEntry(pDecoder, &metaEntry);
+
+  code = metaHandleEntry(pMeta, &metaEntry);
+  if (code) goto _err;
+
+  return code;
+
+_err:
+  metaError("vgId:%d meta snapshot write failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
   return code;
 }
\ No newline at end of file

source/dnode/vnode/src/meta/metaTable.c

@@ -17,7 +17,6 @@
 
 static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
 static int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
-static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME);
 static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME);
 static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME);
 static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME);
@@ -51,7 +50,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
   if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) {
     return -1;
   }
-  void *      data = pCtbEntry->ctbEntry.pTags;
+  void       *data = pCtbEntry->ctbEntry.pTags;
   const char *tagName = pSchema->name;
 
   tb_uid_t    suid = pCtbEntry->ctbEntry.suid;
@@ -70,7 +69,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
     STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i);
     char     type = pTagVal->type;
 
-    char *  key = pTagVal->pKey;
+    char   *key = pTagVal->pKey;
     int32_t nKey = strlen(key);
 
     SIndexTerm *term = NULL;
@@ -78,7 +77,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
       term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0);
     } else if (type == TSDB_DATA_TYPE_NCHAR) {
       if (pTagVal->nData > 0) {
-        char *  val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
+        char   *val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
         int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE);
         memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE);
         type = TSDB_DATA_TYPE_VARCHAR;
@@ -109,7 +108,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche
   if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) {
     return -1;
   }
-  void *      data = pCtbEntry->ctbEntry.pTags;
+  void       *data = pCtbEntry->ctbEntry.pTags;
   const char *tagName = pSchema->name;
 
   tb_uid_t    suid = pCtbEntry->ctbEntry.suid;
@@ -128,7 +127,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche
     STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i);
     char     type = pTagVal->type;
 
-    char *  key = pTagVal->pKey;
+    char   *key = pTagVal->pKey;
     int32_t nKey = strlen(key);
 
     SIndexTerm *term = NULL;
@@ -136,7 +135,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche
       term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0);
     } else if (type == TSDB_DATA_TYPE_NCHAR) {
       if (pTagVal->nData > 0) {
-        char *  val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
+        char   *val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
         int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE);
         memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE);
         type = TSDB_DATA_TYPE_VARCHAR;
@@ -169,9 +168,9 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
   int         vLen = 0;
   const void *pKey = NULL;
   const void *pVal = NULL;
-  void *      pBuf = NULL;
+  void       *pBuf = NULL;
   int32_t     szBuf = 0;
-  void *      p = NULL;
+  void       *p = NULL;
   SMetaReader mr = {0};
 
   // validate req
@@ -229,7 +228,7 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) {
   }
 
   // drop all child tables
-  TBC *   pCtbIdxc = NULL;
+  TBC    *pCtbIdxc = NULL;
   SArray *pArray = taosArrayInit(8, sizeof(tb_uid_t));
 
   tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn);
@@ -285,8 +284,8 @@ _exit:
 int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
   SMetaEntry  oStbEntry = {0};
   SMetaEntry  nStbEntry = {0};
-  TBC *       pUidIdxc = NULL;
-  TBC *       pTbDbc = NULL;
+  TBC        *pUidIdxc = NULL;
+  TBC        *pTbDbc = NULL;
   const void *pData;
   int         nData;
   int64_t     oversion;
@@ -409,7 +408,7 @@ _err:
 }
 
 int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids) {
-  void *   pData = NULL;
+  void    *pData = NULL;
   int      nData = 0;
   int      rc = 0;
   tb_uid_t uid;
@@ -477,7 +476,7 @@ static int metaDeleteTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
 }
 
 static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
-  void *     pData = NULL;
+  void      *pData = NULL;
   int        nData = 0;
   int        rc = 0;
   SMetaEntry e = {0};
@@ -538,14 +537,14 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
 }
 
 static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq, STableMetaRsp *pMetaRsp) {
-  void *          pVal = NULL;
+  void           *pVal = NULL;
   int             nVal = 0;
-  const void *    pData = NULL;
+  const void     *pData = NULL;
   int             nData = 0;
   int             ret = 0;
   tb_uid_t        uid;
   int64_t         oversion;
-  SSchema *       pColumn = NULL;
+  SSchema        *pColumn = NULL;
   SMetaEntry      entry = {0};
   SSchemaWrapper *pSchema;
   int             c;
@@ -699,7 +698,7 @@ _err:
 static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
   SMetaEntry  ctbEntry = {0};
   SMetaEntry  stbEntry = {0};
-  void *      pVal = NULL;
+  void       *pVal = NULL;
   int         nVal = 0;
   int         ret;
   int         c;
@@ -730,7 +729,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
   oversion = *(int64_t *)pData;
 
   // search table.db
-  TBC *    pTbDbc = NULL;
+  TBC     *pTbDbc = NULL;
   SDecoder dc1 = {0};
   SDecoder dc2 = {0};
 
@@ -754,7 +753,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
   metaDecodeEntry(&dc2, &stbEntry);
 
   SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag;
-  SSchema *       pColumn = NULL;
+  SSchema        *pColumn = NULL;
   int32_t         iCol = 0;
   for (;;) {
     pColumn = NULL;
@@ -784,8 +783,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
     memcpy((void *)ctbEntry.ctbEntry.pTags, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal);
   } else {
     const STag *pOldTag = (const STag *)ctbEntry.ctbEntry.pTags;
-    STag *      pNewTag = NULL;
-    SArray *    pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal));
+    STag       *pNewTag = NULL;
+    SArray     *pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal));
     if (!pTagArray) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       goto _err;
@@ -844,7 +843,7 @@ _err:
 }
 
 static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
-  void *      pVal = NULL;
+  void       *pVal = NULL;
   int         nVal = 0;
   const void *pData = NULL;
   int         nData = 0;
@@ -948,8 +947,8 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta
 
 static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) {
   STbDbKey tbDbKey;
-  void *   pKey = NULL;
-  void *   pVal = NULL;
+  void    *pKey = NULL;
+  void    *pVal = NULL;
   int      kLen = 0;
   int      vLen = 0;
   SEncoder coder = {0};
@@ -1055,14 +1054,14 @@ static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) {
 }
 
 static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
-  void *         pData = NULL;
+  void          *pData = NULL;
   int            nData = 0;
   STbDbKey       tbDbKey = {0};
   SMetaEntry     stbEntry = {0};
-  STagIdxKey *   pTagIdxKey = NULL;
+  STagIdxKey    *pTagIdxKey = NULL;
   int32_t        nTagIdxKey;
   const SSchema *pTagColumn;       // = &stbEntry.stbEntry.schema.pSchema[0];
-  const void *   pTagData = NULL;  //
+  const void    *pTagData = NULL;  //
   int32_t        nTagData = 0;
   SDecoder       dc = {0};
 
@@ -1109,7 +1108,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
 
 static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) {
   SEncoder              coder = {0};
-  void *                pVal = NULL;
+  void                 *pVal = NULL;
   int                   vLen = 0;
   int                   rcode = 0;
   SSkmDbKey             skmDbKey = {0};
@@ -1151,7 +1150,7 @@ _exit:
   return rcode;
 }
 
-static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) {
+int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) {
   metaWLock(pMeta);
 
   // save to table.db

source/dnode/vnode/src/sma/sma.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "sma.h"
-
-// functions for external invocation
-
-// TODO: Who is responsible for resource allocate and release?
-int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg) {
-  int32_t code = TSDB_CODE_SUCCESS;
-
-  if ((code = tdProcessTSmaInsertImpl(pSma, indexUid, msg)) < 0) {
-    smaWarn("vgId:%d, insert tsma data failed since %s", SMA_VID(pSma), tstrerror(terrno));
-  }
-  // TODO: destroy SSDataBlocks(msg)
-  return code;
-}
-
-int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg) {
-  int32_t code = TSDB_CODE_SUCCESS;
-
-  if ((code = tdProcessTSmaCreateImpl(pSma, version, msg)) < 0) {
-    smaWarn("vgId:%d, create tsma failed since %s", SMA_VID(pSma), tstrerror(terrno));
-  }
-  // TODO: destroy SSDataBlocks(msg)
-  return code;
-}
-
-int32_t smaGetTSmaDays(SVnodeCfg* pCfg, void* pCont, uint32_t contLen, int32_t* days) {
-  int32_t code = TSDB_CODE_SUCCESS;
-  if ((code = tdProcessTSmaGetDaysImpl(pCfg, pCont, contLen, days)) < 0) {
-    smaWarn("vgId:%d, get tsma days failed since %s", pCfg->vgId, tstrerror(terrno));
-  }
-  smaDebug("vgId:%d, get tsma days %d", pCfg->vgId, *days);
-  return code;
-}
-
-
-// functions for internal invocation
-
-#if 0
-
-/**
- * @brief TODO: Assume that the final generated result it less than 3M
- *
- * @param pReq
- * @param pDataBlocks
- * @param vgId
- * @param suid  // TODO: check with Liao whether suid response is reasonable
- *
- * TODO: colId should be set
- */
-int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId,
-                                    tb_uid_t suid, const char* stbName, bool isCreateCtb) {
-  int32_t sz = taosArrayGetSize(pDataBlocks);
-  int32_t bufSize = sizeof(SSubmitReq);
-  for (int32_t i = 0; i < sz; ++i) {
-    SDataBlockInfo* pBlkInfo = &((SSDataBlock*)taosArrayGet(pDataBlocks, i))->info;
-    bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(pBlkInfo->numOfCols));
-    bufSize += sizeof(SSubmitBlk);
-  }
-
-  *pReq = taosMemoryCalloc(1, bufSize);
-  if (!(*pReq)) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return TSDB_CODE_FAILED;
-  }
-  void* pDataBuf = *pReq;
-
-  SArray*     pTagArray = NULL;
-  int32_t     msgLen = sizeof(SSubmitReq);
-  int32_t     numOfBlks = 0;
-  int32_t     schemaLen = 0;
-  SRowBuilder rb = {0};
-  tdSRowInit(&rb, pTSchema->version);
-
-  for (int32_t i = 0; i < sz; ++i) {
-    SSDataBlock*    pDataBlock = taosArrayGet(pDataBlocks, i);
-    SDataBlockInfo* pDataBlkInfo = &pDataBlock->info;
-    int32_t         colNum = pDataBlkInfo->numOfCols;
-    int32_t         rows = pDataBlkInfo->rows;
-    int32_t         rowSize = pDataBlkInfo->rowSize;
-    int64_t         groupId = pDataBlkInfo->groupId;
-
-    if (rb.nCols != colNum) {
-      tdSRowSetTpInfo(&rb, colNum, pTSchema->flen);
-    }
-
-   if(isCreateCtb) {
-     SMetaReader  mr = {0};
-     const char*  ctbName = buildCtbNameByGroupId(stbName, pDataBlock->info.groupId);
-     if (metaGetTableEntryByName(&mr, ctbName) != 0) {
-       smaDebug("vgId:%d, no tsma ctb %s exists", vgId, ctbName);
-     }
-     SVCreateTbReq ctbReq = {0};
-     ctbReq.name = ctbName;
-     ctbReq.type = TSDB_CHILD_TABLE;
-     ctbReq.ctb.suid = suid;
-
-     STagVal tagVal = {.cid = colNum + PRIMARYKEY_TIMESTAMP_COL_ID,
-                       .type = TSDB_DATA_TYPE_BIGINT,
-                       .i64 = groupId};
-     STag*   pTag = NULL;
-     if(!pTagArray) {
-       pTagArray = taosArrayInit(1, sizeof(STagVal));
-       if (!pTagArray) goto _err;
-     }
-     taosArrayClear(pTagArray);
-     taosArrayPush(pTagArray, &tagVal);
-     tTagNew(pTagArray, 1, false, &pTag);
-     if (pTag == NULL) {
-       tdDestroySVCreateTbReq(&ctbReq);
-       goto _err;
-      }
-      ctbReq.ctb.pTag = (uint8_t*)pTag;
-
-      int32_t code;
-      tEncodeSize(tEncodeSVCreateTbReq, &ctbReq, schemaLen, code);
-
-      tdDestroySVCreateTbReq(&ctbReq);
-      if (code < 0) {
-        goto _err;
-      }
-   }
-
-
-
-    SSubmitBlk* pSubmitBlk = POINTER_SHIFT(pDataBuf, msgLen);
-    pSubmitBlk->suid = suid;
-    pSubmitBlk->uid = groupId;
-    pSubmitBlk->numOfRows = rows;
-
-    msgLen += sizeof(SSubmitBlk);
-    int32_t dataLen = 0;
-    for (int32_t j = 0; j < rows; ++j) {                     // iterate by row
-      tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen));  // set row buf
-      bool    isStartKey = false;
-      int32_t offset = 0;
-      for (int32_t k = 0; k < colNum; ++k) {  // iterate by column
-        SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
-        STColumn*        pCol = &pTSchema->columns[k];
-        void*            var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
-        switch (pColInfoData->info.type) {
-          case TSDB_DATA_TYPE_TIMESTAMP:
-            if (!isStartKey) {
-              isStartKey = true;
-              tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true,
-                                  offset, k);
-
-            } else {
-              tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var,
-                                  true, offset, k);
-            }
-            break;
-          case TSDB_DATA_TYPE_NCHAR: {
-            tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_NCHAR, TD_VTYPE_NORM, var, true,
-                                offset, k);
-            break;
-          }
-          case TSDB_DATA_TYPE_VARCHAR: {  // TSDB_DATA_TYPE_BINARY
-            tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_VARCHAR, TD_VTYPE_NORM, var, true,
-                                offset, k);
-            break;
-          }
-          case TSDB_DATA_TYPE_VARBINARY:
-          case TSDB_DATA_TYPE_DECIMAL:
-          case TSDB_DATA_TYPE_BLOB:
-          case TSDB_DATA_TYPE_JSON:
-          case TSDB_DATA_TYPE_MEDIUMBLOB:
-            uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type);
-            TASSERT(0);
-            break;
-          default:
-            if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) {
-              if (pCol->type == pColInfoData->info.type) {
-                tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, var, true, offset,
-                                    k);
-              } else {
-                char tv[8] = {0};
-                if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) {
-                  float v = 0;
-                  GET_TYPED_DATA(v, float, pColInfoData->info.type, var);
-                  SET_TYPED_DATA(&tv, pCol->type, v);
-                } else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) {
-                  double v = 0;
-                  GET_TYPED_DATA(v, double, pColInfoData->info.type, var);
-                  SET_TYPED_DATA(&tv, pCol->type, v);
-                } else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) {
-                  int64_t v = 0;
-                  GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var);
-                  SET_TYPED_DATA(&tv, pCol->type, v);
-                } else {
-                  uint64_t v = 0;
-                  GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var);
-                  SET_TYPED_DATA(&tv, pCol->type, v);
-                }
-                tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, tv, true, offset,
-                                    k);
-              }
-            } else {
-              uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type);
-              TASSERT(0);
-            }
-            break;
-        }
-        offset += TYPE_BYTES[pCol->type];  // sum/avg would convert to int64_t/uint64_t/double during aggregation
-      }
-      dataLen += TD_ROW_LEN(rb.pBuf);
-#ifdef TD_DEBUG_PRINT_ROW
-      tdSRowPrint(rb.pBuf, pTSchema, __func__);
-#endif
-    }
-
-    ++numOfBlks;
-
-    pSubmitBlk->dataLen = dataLen;
-    msgLen += pSubmitBlk->dataLen;
-  }
-
-  (*pReq)->length = msgLen;
-
-  (*pReq)->header.vgId = htonl(vgId);
-  (*pReq)->header.contLen = htonl(msgLen);
-  (*pReq)->length = (*pReq)->header.contLen;
-  (*pReq)->numOfBlocks = htonl(numOfBlks);
-  SSubmitBlk* blk = (SSubmitBlk*)((*pReq) + 1);
-  while (numOfBlks--) {
-    int32_t dataLen = blk->dataLen;
-    blk->uid = htobe64(blk->uid);
-    blk->suid = htobe64(blk->suid);
-    blk->padding = htonl(blk->padding);
-    blk->sversion = htonl(blk->sversion);
-    blk->dataLen = htonl(blk->dataLen);
-    blk->schemaLen = htonl(blk->schemaLen);
-    blk->numOfRows = htons(blk->numOfRows);
-    blk = (SSubmitBlk*)(blk->data + dataLen);
-  }
-  return TSDB_CODE_SUCCESS;
-_err:
-  taosMemoryFreeClear(*pReq);
-  taosArrayDestroy(pTagArray);
-
-  return TSDB_CODE_FAILED;
-}
-#endif

source/dnode/vnode/src/sma/smaCommit.c

@@ -121,7 +121,7 @@ static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma) {
   // step 3: perform persist task for qTaskInfo
   tdRSmaPersistExecImpl(pRSmaStat);
 
-  smaDebug("vgId:%d, rsma pre commit succeess", SMA_VID(pSma));
+  smaDebug("vgId:%d, rsma pre commit success", SMA_VID(pSma));
 
   return TSDB_CODE_SUCCESS;
 }
@@ -173,6 +173,7 @@ static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma) {
   }
 
   if ((pDir = taosOpenDir(dir)) == NULL) {
+    regfree(&regex);
     terrno = TAOS_SYSTEM_ERROR(errno);
     smaWarn("vgId:%d, rsma post commit, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr());
     return TSDB_CODE_FAILED;

source/dnode/vnode/src/sma/smaEnv.c

@@ -18,7 +18,7 @@
 typedef struct SSmaStat SSmaStat;
 
 #define RSMA_TASK_INFO_HASH_SLOT 8
-#define SMA_MGMT_REF_NUM         1024
+#define SMA_MGMT_REF_NUM         10240
 
 extern SSmaMgmt smaMgmt;
 
@@ -30,7 +30,73 @@ static int32_t  tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SSmaE
 static void    *tdFreeTSmaStat(STSmaStat *pStat);
 static void     tdDestroyRSmaStat(void *pRSmaStat);
 
+/**
+ * @brief rsma init
+ *
+ * @return int32_t
+ */
 // implementation
+int32_t smaInit() {
+  int8_t  old;
+  int32_t nLoops = 0;
+  while (1) {
+    old = atomic_val_compare_exchange_8(&smaMgmt.inited, 0, 2);
+    if (old != 2) break;
+    if (++nLoops > 1000) {
+      sched_yield();
+      nLoops = 0;
+    }
+  }
+
+  if (old == 0) {
+    // init tref rset
+    smaMgmt.rsetId = taosOpenRef(SMA_MGMT_REF_NUM, tdDestroyRSmaStat);
+
+    if (smaMgmt.rsetId < 0) {
+      atomic_store_8(&smaMgmt.inited, 0);
+      smaError("failed to init sma rset since %s", terrstr());
+      return TSDB_CODE_FAILED;
+    }
+
+    // init fetch timer handle
+    smaMgmt.tmrHandle = taosTmrInit(10000, 100, 10000, "RSMA");
+    if (!smaMgmt.tmrHandle) {
+      taosCloseRef(smaMgmt.rsetId);
+      atomic_store_8(&smaMgmt.inited, 0);
+      smaError("failed to init sma tmr hanle since %s", terrstr());
+      return TSDB_CODE_FAILED;
+    }
+
+    atomic_store_8(&smaMgmt.inited, 1);
+    smaInfo("sma mgmt env is initialized, rsetId:%d, tmrHandle:%p", smaMgmt.rsetId, smaMgmt.tmrHandle);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief rsma cleanup
+ *
+ */
+void smaCleanUp() {
+  int8_t  old;
+  int32_t nLoops = 0;
+  while (1) {
+    old = atomic_val_compare_exchange_8(&smaMgmt.inited, 1, 2);
+    if (old != 2) break;
+    if (++nLoops > 1000) {
+      sched_yield();
+      nLoops = 0;
+    }
+  }
+
+  if (old == 1) {
+    taosCloseRef(smaMgmt.rsetId);
+    taosTmrCleanUp(smaMgmt.tmrHandle);
+    smaInfo("sma mgmt env is cleaned up, rsetId:%d, tmrHandle:%p", smaMgmt.rsetId, smaMgmt.tmrHandle);
+    atomic_store_8(&smaMgmt.inited, 0);
+  }
+}
 
 static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path) {
   SSmaEnv *pEnv = NULL;
@@ -135,34 +201,24 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS
       atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_INIT);
 
       // init smaMgmt
-      smaMgmt.smaRef = taosOpenRef(SMA_MGMT_REF_NUM, tdDestroyRSmaStat);
-      if (smaMgmt.smaRef < 0) {
-        smaError("init smaRef failed, num:%d", SMA_MGMT_REF_NUM);
-        terrno = TSDB_CODE_OUT_OF_MEMORY;
-        return TSDB_CODE_FAILED;
-      }
+      smaInit();
 
-      int64_t refId = taosAddRef(smaMgmt.smaRef, pRSmaStat);
+      int64_t refId = taosAddRef(smaMgmt.rsetId, pRSmaStat);
       if (refId < 0) {
-        smaError("taosAddRef smaRef failed, since:%s", tstrerror(terrno));
+        smaError("vgId:%d, taosAddRef refId:%" PRIi64 " to rsetId rsetId:%d max:%d failed since:%s", SMA_VID(pSma),
+                 refId, smaMgmt.rsetId, SMA_MGMT_REF_NUM, tstrerror(terrno));
         return TSDB_CODE_FAILED;
+      } else {
+        smaDebug("vgId:%d, taosAddRef refId:%" PRIi64 " to rsetId rsetId:%d max:%d succeed", SMA_VID(pSma), refId,
+                 smaMgmt.rsetId, SMA_MGMT_REF_NUM);
       }
       pRSmaStat->refId = refId;
 
-      // init timer
-      RSMA_TMR_HANDLE(pRSmaStat) = taosTmrInit(10000, 100, 10000, "RSMA");
-      if (!RSMA_TMR_HANDLE(pRSmaStat)) {
-        taosMemoryFreeClear(*pSmaStat);
-        return TSDB_CODE_FAILED;
-      }
 
       // init hash
       RSMA_INFO_HASH(pRSmaStat) = taosHashInit(
           RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
       if (!RSMA_INFO_HASH(pRSmaStat)) {
-        if (RSMA_TMR_HANDLE(pRSmaStat)) {
-          taosTmrCleanUp(RSMA_TMR_HANDLE(pRSmaStat));
-        }
         taosMemoryFreeClear(*pSmaStat);
         return TSDB_CODE_FAILED;
       }
@@ -223,7 +279,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
       void *infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), NULL);
       while (infoHash) {
         SRSmaInfo *pSmaInfo = *(SRSmaInfo **)infoHash;
-        tdFreeRSmaInfo(pSmaInfo);
+        tdFreeRSmaInfo(pSma, pSmaInfo);
         infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), infoHash);
       }
     }
@@ -244,11 +300,6 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
         nLoops = 0;
       }
     }
-
-    // step 6: cleanup the timer handle
-    if (RSMA_TMR_HANDLE(pStat)) {
-      taosTmrCleanUp(RSMA_TMR_HANDLE(pStat));
-    }
   }
 }
 
@@ -275,8 +326,13 @@ int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) {
       tdDestroyTSmaStat(SMA_TSMA_STAT(pSmaStat));
     } else if (smaType == TSDB_SMA_TYPE_ROLLUP) {
       SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSmaStat);
-      if (taosRemoveRef(smaMgmt.smaRef, RSMA_REF_ID(pRSmaStat)) < 0) {
-        smaError("remove refId from rsmaRef:0x%" PRIx64 " failed since %s", RSMA_REF_ID(pRSmaStat), terrstr());
+      if (taosRemoveRef(smaMgmt.rsetId, RSMA_REF_ID(pRSmaStat)) < 0) {
+        smaError("vgId:%d, remove refId:%" PRIi64 " from rsmaRef:%" PRIi32 " failed since %s", SMA_VID(pRSmaStat->pSma),
+                 RSMA_REF_ID(pRSmaStat), smaMgmt.rsetId, terrstr());
+        ASSERT(0);
+      } else {
+        smaDebug("vgId:%d, remove refId:%" PRIi64 " from rsmaRef:%" PRIi32 " succeed", SMA_VID(pRSmaStat->pSma),
+                 RSMA_REF_ID(pRSmaStat), smaMgmt.rsetId);
       }
     } else {
       ASSERT(0);
@@ -323,7 +379,7 @@ int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) {
       }
       break;
     default:
-      TASSERT(0);
+      smaError("vgId:%d undefined smaType:%", SMA_VID(pSma), smaType);
       return TSDB_CODE_FAILED;
   }
 

source/dnode/vnode/src/sma/smaRollup.c

@@ -19,23 +19,27 @@
 #define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t))  // len + type + suid
 
 SSmaMgmt smaMgmt = {
-    .smaRef = -1,
+    .inited = 0,
+    .rsetId = -1,
 };
 
 #define TD_QTASKINFO_FNAME_PREFIX "qtaskinfo.ver"
 typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem;
 typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter;
 
-static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
-static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids);
-static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo *pRSmaInfo, SReadHandle *handle,
-                                       int8_t idx);
-static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *rsmaItem,
-                                 tb_uid_t suid, int8_t level);
-static void    tdRSmaFetchTrigger(void *param, void *tmrId);
-static void    tdRSmaPersistTrigger(void *param, void *tmrId);
-static void   *tdRSmaPersistExec(void *param);
-static void    tdRSmaQTaskInfoGetFName(int32_t vid, int64_t version, char *outputName);
+static int32_t    tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
+static int32_t    tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids);
+static int32_t    tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo,
+                                          SReadHandle *handle, int8_t idx);
+static int32_t    tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *rsmaItem,
+                                    STSchema *pTSchema, tb_uid_t suid, int8_t level);
+static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid);
+static int32_t    tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, SRSmaStat *pStat,
+                                             int8_t blkType);
+static void       tdRSmaFetchTrigger(void *param, void *tmrId);
+static void       tdRSmaPersistTrigger(void *param, void *tmrId);
+static void      *tdRSmaPersistExec(void *param);
+static void       tdRSmaQTaskInfoGetFName(int32_t vid, int64_t version, char *outputName);
 
 static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile);
 static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish);
@@ -47,25 +51,26 @@ static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed);
 static int32_t tdRSmaRestoreTSDataReload(SSma *pSma, int64_t committed);
 
 struct SRSmaInfoItem {
-  SRSmaInfo *pRsmaInfo;
-  int64_t    refId;
-  void      *taskInfo;  // qTaskInfo_t
-  tmr_h      tmrId;
-  int8_t     level;
-  int8_t     tmrInitFlag;
-  int8_t     triggerStat;
-  int32_t    maxDelay;
+  void   *taskInfo;  // qTaskInfo_t
+  int64_t refId;
+  tmr_h   tmrId;
+  int32_t maxDelay;
+  int8_t  level;
+  int8_t  triggerStat;
 };
 
 struct SRSmaInfo {
   STSchema     *pTSchema;
-  SRSmaStat    *pStat;
   int64_t       suid;
   SRSmaInfoItem items[TSDB_RETENTION_L2];
 };
 
-#define RSMA_INFO_SMA(r)  ((r)->pStat->pSma)
-#define RSMA_INFO_STAT(r) ((r)->pStat)
+static SRSmaInfo *tdGetRSmaInfoByItem(SRSmaInfoItem *pItem) {
+  // adapt accordingly if definition of SRSmaInfo update
+  int32_t rsmaInfoHeadLen = sizeof(int64_t) + sizeof(STSchema *);
+  ASSERT(pItem->level == 1 || pItem->level == 2);
+  return (SRSmaInfo *)POINTER_SHIFT(pItem, -sizeof(SRSmaInfoItem) * (pItem->level - 1) - rsmaInfoHeadLen);
+}
 
 struct SRSmaQTaskInfoItem {
   int32_t len;
@@ -107,9 +112,8 @@ static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle, int32_t vgId,
   }
 }
 
-void *tdFreeRSmaInfo(SRSmaInfo *pInfo) {
+void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) {
   if (pInfo) {
-    SSma *pSma = RSMA_INFO_SMA(pInfo);
     for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
       SRSmaInfoItem *pItem = &pInfo->items[i];
       if (pItem->taskInfo) {
@@ -142,8 +146,6 @@ static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) {
 }
 
 static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) {
-  SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
-  SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
   SRSmaInfo *pRSmaInfo = NULL;
 
   if (!suid || !tbUids) {
@@ -152,8 +154,9 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids)
     return TSDB_CODE_FAILED;
   }
 
-  pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), suid, sizeof(tb_uid_t));
-  if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
+  pRSmaInfo = tdGetRSmaInfoBySuid(pSma, *suid);
+
+  if (!pRSmaInfo) {
     smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid);
     terrno = TSDB_CODE_RSMA_INVALID_STAT;
     return TSDB_CODE_FAILED;
@@ -251,15 +254,14 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo *pRSmaInfo, SReadHandle *pReadHandle,
-                                       int8_t idx) {
+static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo,
+                                       SReadHandle *pReadHandle, int8_t idx) {
   SRetention *pRetention = SMA_RETENTION(pSma);
   STsdbCfg   *pTsdbCfg = SMA_TSDB_CFG(pSma);
 
   if (param->qmsg[idx]) {
     SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]);
-    pItem->refId = RSMA_REF_ID(pRSmaInfo->pStat);
-    pItem->pRsmaInfo = pRSmaInfo;
+    pItem->refId = RSMA_REF_ID(pStat);
     pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], pReadHandle);
     if (!pItem->taskInfo) {
       terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE;
@@ -347,14 +349,13 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
     goto _err;
   }
   pRSmaInfo->pTSchema = pTSchema;
-  pRSmaInfo->pStat = pStat;
   pRSmaInfo->suid = suid;
 
-  if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 0) < 0) {
+  if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, &handle, 0) < 0) {
     goto _err;
   }
 
-  if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 1) < 0) {
+  if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, &handle, 1) < 0) {
     goto _err;
   }
 
@@ -366,7 +367,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
 
   return TSDB_CODE_SUCCESS;
 _err:
-  tdFreeRSmaInfo(pRSmaInfo);
+  tdFreeRSmaInfo(pSma, pRSmaInfo);
   taosMemoryFree(pReader);
   return TSDB_CODE_FAILED;
 }
@@ -537,10 +538,10 @@ int64_t tdRSmaGetMaxSubmitVer(SSma *pSma, int8_t level) {
   return atomic_load_64(&pRSmaStat->submitVer);
 }
 
-static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) {
-  SArray    *pResult = NULL;
-  SRSmaInfo *pRSmaInfo = pItem->pRsmaInfo;
-  SSma      *pSma = RSMA_INFO_SMA(pRSmaInfo);
+static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, SRSmaStat *pStat,
+                                          int8_t blkType) {
+  SArray *pResult = NULL;
+  SSma   *pSma = pStat->pSma;
 
   while (1) {
     SSDataBlock *output = NULL;
@@ -572,16 +573,16 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType)
     STsdb      *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2);
     SSubmitReq *pReq = NULL;
     // TODO: the schema update should be handled
-    if (buildSubmitReqFromDataBlock(&pReq, pResult, pRSmaInfo->pTSchema, SMA_VID(pSma), pRSmaInfo->suid) < 0) {
+    if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, SMA_VID(pSma), suid) < 0) {
       smaError("vgId:%d, build submit req for rsma table %" PRIi64 "l evel %" PRIi8 " failed since %s", SMA_VID(pSma),
-               pRSmaInfo->suid, pItem->level, terrstr());
+               suid, pItem->level, terrstr());
       goto _err;
     }
 
-    if (pReq && tdProcessSubmitReq(sinkTsdb, atomic_add_fetch_64(&pRSmaInfo->pStat->submitVer, 1), pReq) < 0) {
+    if (pReq && tdProcessSubmitReq(sinkTsdb, atomic_add_fetch_64(&pStat->submitVer, 1), pReq) < 0) {
       taosMemoryFreeClear(pReq);
       smaError("vgId:%d, process submit req for rsma table %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma),
-               pRSmaInfo->suid, pItem->level, terrstr());
+               suid, pItem->level, terrstr());
       goto _err;
     }
 
@@ -599,81 +600,16 @@ _err:
   return TSDB_CODE_FAILED;
 }
 
-/**
- * @brief trigger to get rsma result
- *
- * @param param
- * @param tmrId
- */
-static void tdRSmaFetchTrigger(void *param, void *tmrId) {
-  SRSmaInfoItem *pItem = param;
-  SSma          *pSma = NULL;
-  SRSmaStat     *pStat = (SRSmaStat *)taosAcquireRef(smaMgmt.smaRef, pItem->refId);
-  if (!pStat) {
-    smaDebug("rsma fetch task not start since already destroyed");
-    return;
-  }
-
-  pSma = RSMA_INFO_SMA(pItem->pRsmaInfo);
-
-  // if rsma trigger stat in paused, cancelled or finished, not start fetch task
-  int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat));
-  switch (rsmaTriggerStat) {
-    case TASK_TRIGGER_STAT_PAUSED:
-    case TASK_TRIGGER_STAT_CANCELLED:
-    case TASK_TRIGGER_STAT_FINISHED: {
-      taosReleaseRef(smaMgmt.smaRef, pItem->refId);
-      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is cancelled",
-               SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
-      return;
-    }
-    default:
-      break;
-  }
-
-  int8_t fetchTriggerStat =
-      atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
-  switch (fetchTriggerStat) {
-    case TASK_TRIGGER_STAT_ACTIVE: {
-      smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma),
-               pItem->level, pItem->pRsmaInfo->suid);
-
-      tdRefSmaStat(pSma, (SSmaStat *)pStat);
-
-      SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
-      qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false);
-      tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_BLOCK);
-
-      tdUnRefSmaStat(pSma, (SSmaStat *)pStat);
-    } break;
-    case TASK_TRIGGER_STAT_PAUSED: {
-      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused",
-               SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
-    } break;
-    case TASK_TRIGGER_STAT_INACTIVE: {
-      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is inactive",
-               SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
-    } break;
-    case TASK_TRIGGER_STAT_INIT: {
-      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is init", SMA_VID(pSma),
-               pItem->level, pItem->pRsmaInfo->suid);
-    } break;
-    default: {
-      smaWarn("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is unknown",
-              SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
-    } break;
-  }
-
-_end:
-  taosReleaseRef(smaMgmt.smaRef, pItem->refId);
-}
-
-static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, tb_uid_t suid,
-                                 int8_t level) {
+static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem,
+                                 STSchema *pTSchema, tb_uid_t suid, int8_t level) {
   if (!pItem || !pItem->taskInfo) {
     smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid);
     return TSDB_CODE_SUCCESS;
   }
+  if (!pTSchema) {
+    smaWarn("vgId:%d, no schema to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid);
+    return TSDB_CODE_FAILED;
+  }
 
   smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level,
            pItem->taskInfo, suid);
@@ -683,14 +619,14 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType
     return TSDB_CODE_FAILED;
   }
 
-  tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_SUBMIT);
-  atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE);
-
   SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
   SRSmaStat *pStat = SMA_RSMA_STAT(pEnv->pStat);
 
-  if (pStat->tmrHandle) {
-    taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, pStat->tmrHandle, &pItem->tmrId);
+  tdRSmaFetchAndSubmitResult(pItem, pTSchema, suid, pStat, STREAM_INPUT__DATA_SUBMIT);
+  atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE);
+
+  if (smaMgmt.tmrHandle) {
+    taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, smaMgmt.tmrHandle, &pItem->tmrId);
   } else {
     ASSERT(0);
   }
@@ -698,19 +634,29 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) {
-  SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
+static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) {
+  SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
+  SRSmaStat *pStat = NULL;
   if (!pEnv) {
     // only applicable when rsma env exists
-    return TSDB_CODE_SUCCESS;
+    return NULL;
   }
 
-  SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
-  SRSmaInfo *pRSmaInfo = NULL;
-
-  pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
+  pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
+  if (!pStat || !RSMA_INFO_HASH(pStat)) {
+    return NULL;
+  }
 
+  SRSmaInfo *pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
   if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
+    return NULL;
+  }
+  return pRSmaInfo;
+}
+
+static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) {
+  SRSmaInfo *pRSmaInfo = tdGetRSmaInfoBySuid(pSma, suid);
+  if (!pRSmaInfo) {
     smaDebug("vgId:%d, return as no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid);
     return TSDB_CODE_SUCCESS;
   }
@@ -721,8 +667,8 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb
   }
 
   if (inputType == STREAM_INPUT__DATA_SUBMIT) {
-    tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[0], suid, TSDB_RETENTION_L1);
-    tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[1], suid, TSDB_RETENTION_L2);
+    tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[0], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L1);
+    tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[1], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L2);
   }
 
   return TSDB_CODE_SUCCESS;
@@ -935,13 +881,11 @@ _err:
 }
 
 static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *pItem) {
-  SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT((SSmaEnv *)pSma->pRSmaEnv);
   SRSmaInfo *pRSmaInfo = NULL;
   void      *qTaskInfo = NULL;
 
-  pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &pItem->suid, sizeof(pItem->suid));
-
-  if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
+  pRSmaInfo = tdGetRSmaInfoBySuid(pSma, pItem->suid);
+  if (!pRSmaInfo) {
     smaDebug("vgId:%d, no restore as no rsma info for table:%" PRIu64, SMA_VID(pSma), pItem->suid);
     return TSDB_CODE_SUCCESS;
   }
@@ -1258,7 +1202,8 @@ _end:
   }
 
   atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
-  taosReleaseRef(smaMgmt.smaRef, pRSmaStat->refId);
+  smaDebug("vgId:%d, release rsetId rsetId:%" PRIi64 " refId:%d", SMA_VID(pSma), smaMgmt.rsetId, pRSmaStat->refId);
+  tdReleaseSmaRef(smaMgmt.rsetId, pRSmaStat->refId, __func__, __LINE__);
   taosThreadExit(NULL);
   return NULL;
 }
@@ -1283,7 +1228,9 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) {
               atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)));
     }
     atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
-    taosReleaseRef(smaMgmt.smaRef, pRSmaStat->refId);
+    smaDebug("vgId:%d, release rsetId rsetId:%" PRIi64 " refId:%d)", SMA_VID(pRSmaStat->pSma), smaMgmt.rsetId,
+             pRSmaStat->refId);
+    tdReleaseSmaRef(smaMgmt.rsetId, pRSmaStat->refId, __func__, __LINE__);
   }
 
   taosThreadAttrDestroy(&thAttr);
@@ -1297,8 +1244,8 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) {
  */
 static void tdRSmaPersistTrigger(void *param, void *tmrId) {
   SRSmaStat *rsmaStat = param;
-  SRSmaStat *pRSmaStat = (SRSmaStat *)taosAcquireRef(smaMgmt.smaRef, rsmaStat->refId);
-
+  SRSmaStat *pRSmaStat = (SRSmaStat *)taosAcquireRef(smaMgmt.rsetId, rsmaStat->refId);
+  ASSERT(0);
   if (!pRSmaStat) {
     smaDebug("rsma persistence task not start since already destroyed");
     return;
@@ -1341,5 +1288,81 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) {
       smaWarn("rsma persistence not start since unknown stat %" PRIi8, tmrStat);
     } break;
   }
-  taosReleaseRef(smaMgmt.smaRef, rsmaStat->refId);
+  taosReleaseRef(smaMgmt.rsetId, rsmaStat->refId);
+}
+
+/**
+ * @brief trigger to get rsma result
+ *
+ * @param param
+ * @param tmrId
+ */
+static void tdRSmaFetchTrigger(void *param, void *tmrId) {
+  SRSmaInfoItem *pItem = param;
+  SSma          *pSma = NULL;
+  SRSmaStat     *pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__);
+
+  if (!pStat) {
+    smaDebug("rsma fetch task not start since already destroyed, rsetId rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId,
+             pItem->refId);
+    return;
+  }
+
+  pSma = pStat->pSma;
+
+  // if rsma trigger stat in paused, cancelled or finished, not start fetch task
+  int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat));
+  switch (rsmaTriggerStat) {
+    case TASK_TRIGGER_STAT_PAUSED:
+    case TASK_TRIGGER_STAT_CANCELLED:
+    case TASK_TRIGGER_STAT_FINISHED: {
+      tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__);
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data since stat is %" PRIi8 ", rsetId rsetId:%" PRIi64
+               " refId:%d",
+               SMA_VID(pSma), pItem->level, rsmaTriggerStat, smaMgmt.rsetId, pItem->refId);
+      return;
+    }
+    default:
+      break;
+  }
+
+  SRSmaInfo *pRSmaInfo = tdGetRSmaInfoByItem(pItem);
+
+  ASSERT(pRSmaInfo->suid > 0);
+
+  int8_t fetchTriggerStat =
+      atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
+  switch (fetchTriggerStat) {
+    case TASK_TRIGGER_STAT_ACTIVE: {
+      smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma),
+               pItem->level, pRSmaInfo->suid);
+
+      tdRefSmaStat(pSma, (SSmaStat *)pStat);
+
+      SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
+      qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false);
+      tdRSmaFetchAndSubmitResult(pItem, pRSmaInfo->pTSchema, pRSmaInfo->suid, pStat, STREAM_INPUT__DATA_BLOCK);
+
+      tdUnRefSmaStat(pSma, (SSmaStat *)pStat);
+    } break;
+    case TASK_TRIGGER_STAT_PAUSED: {
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused",
+               SMA_VID(pSma), pItem->level, pRSmaInfo->suid);
+    } break;
+    case TASK_TRIGGER_STAT_INACTIVE: {
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is inactive",
+               SMA_VID(pSma), pItem->level, pRSmaInfo->suid);
+    } break;
+    case TASK_TRIGGER_STAT_INIT: {
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is init", SMA_VID(pSma),
+               pItem->level, pRSmaInfo->suid);
+    } break;
+    default: {
+      smaWarn("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is unknown",
+              SMA_VID(pSma), pItem->level, pRSmaInfo->suid);
+    } break;
+  }
+
+_end:
+  tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__);
 }

source/dnode/vnode/src/sma/smaTimeRange.c

@@ -20,6 +20,36 @@
 #define SMA_STORAGE_MINUTES_DAY  1440
 #define SMA_STORAGE_SPLIT_FACTOR 14400  // least records in tsma file
 
+// TODO: Who is responsible for resource allocate and release?
+int32_t tdProcessTSmaInsert(SSma *pSma, int64_t indexUid, const char *msg) {
+  int32_t code = TSDB_CODE_SUCCESS;
+
+  if ((code = tdProcessTSmaInsertImpl(pSma, indexUid, msg)) < 0) {
+    smaWarn("vgId:%d, insert tsma data failed since %s", SMA_VID(pSma), tstrerror(terrno));
+  }
+  // TODO: destroy SSDataBlocks(msg)
+  return code;
+}
+
+int32_t tdProcessTSmaCreate(SSma *pSma, int64_t version, const char *msg) {
+  int32_t code = TSDB_CODE_SUCCESS;
+
+  if ((code = tdProcessTSmaCreateImpl(pSma, version, msg)) < 0) {
+    smaWarn("vgId:%d, create tsma failed since %s", SMA_VID(pSma), tstrerror(terrno));
+  }
+  // TODO: destroy SSDataBlocks(msg)
+  return code;
+}
+
+int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) {
+  int32_t code = TSDB_CODE_SUCCESS;
+  if ((code = tdProcessTSmaGetDaysImpl(pCfg, pCont, contLen, days)) < 0) {
+    smaWarn("vgId:%d, get tsma days failed since %s", pCfg->vgId, tstrerror(terrno));
+  }
+  smaDebug("vgId:%d, get tsma days %d", pCfg->vgId, *days);
+  return code;
+}
+
 /**
  * @brief Judge the tsma file split days
  *

source/dnode/vnode/src/sma/smaUtil.c

@@ -294,4 +294,23 @@ int32_t tdRemoveTFile(STFile *pTFile) {
 }
 
 // smaXXXUtil ================
+void *tdAcquireSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln) {
+  void *pResult = taosAcquireRef(rsetId, refId);
+  if (!pResult) {
+    smaWarn("%s:%d taosAcquireRef for rsetId:%" PRIi64 " refId:%d failed since %s", tags, ln, rsetId, refId, terrstr());
+  } else {
+    smaDebug("%s:%d taosAcquireRef for rsetId:%" PRIi64 " refId:%d success", tags, ln, rsetId, refId);
+  }
+  return pResult;
+}
+
+int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln) {
+  if (taosReleaseRef(rsetId, refId) < 0) {
+    smaWarn("%s:%d taosReleaseRef for rsetId:%" PRIi64 " refId:%d failed since %s", tags, ln, rsetId, refId, terrstr());
+    return TSDB_CODE_FAILED;
+  }
+  smaDebug("%s:%d taosReleaseRef for rsetId:%" PRIi64 " refId:%d success", tags, ln, rsetId, refId);
+
+  return TSDB_CODE_SUCCESS;
+}
 // ...
\ No newline at end of file

source/dnode/vnode/src/tq/tq.c

@@ -112,7 +112,8 @@ int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq,
   };
   tmsgSendRsp(&resp);
 
-  tqDebug("vg %d from consumer %ld (epoch %d) send rsp, res msg type %d, reqOffset: %ld, rspOffset: %ld",
+  tqDebug("vgId:%d from consumer:%" PRId64 ", (epoch %d) send rsp, res msg type %d, reqOffset:%" PRId64
+          ", rspOffset:%" PRId64,
           TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->resMsgType, pRsp->reqOffset, pRsp->rspOffset);
 
   return 0;
@@ -162,7 +163,7 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con
   char buf2[80];
   tFormatOffset(buf1, 80, &pRsp->reqOffset);
   tFormatOffset(buf2, 80, &pRsp->rspOffset);
-  tqDebug("vg %d from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %s, rspOffset: %s",
+  tqDebug("vgId:%d from consumer:%" PRId64 ", (epoch %d) send rsp, block num: %d, reqOffset:%s, rspOffset:%s",
           TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2);
 
   return 0;
@@ -179,10 +180,10 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) {
   tDecoderClear(&decoder);
 
   if (offset.val.type == TMQ_OFFSET__SNAPSHOT_DATA) {
-    tqDebug("receive offset commit msg to %s on vg %d, offset(type:snapshot) uid: %ld, ts: %ld", offset.subKey,
-            TD_VID(pTq->pVnode), offset.val.uid, offset.val.ts);
+    tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:snapshot) uid:%" PRId64 ", ts:%" PRId64,
+            offset.subKey, TD_VID(pTq->pVnode), offset.val.uid, offset.val.ts);
   } else if (offset.val.type == TMQ_OFFSET__LOG) {
-    tqDebug("receive offset commit msg to %s on vg %d, offset(type:log) version: %ld", offset.subKey,
+    tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, offset.subKey,
             TD_VID(pTq->pVnode), offset.val.version);
   } else {
     ASSERT(0);
@@ -244,22 +245,18 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   STqOffsetVal fetchOffsetNew;
 
   // 1.find handle
-  char buf[80];
-  tFormatOffset(buf, 80, &reqOffset);
-  tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req offset %s", consumerId, pReq->epoch,
-          TD_VID(pTq->pVnode), buf);
-
   STqHandle* pHandle = taosHashGet(pTq->handles, pReq->subKey, strlen(pReq->subKey));
   /*ASSERT(pHandle);*/
   if (pHandle == NULL) {
-    tqError("tmq poll: no consumer handle for consumer %ld in vg %d, subkey %s", consumerId, TD_VID(pTq->pVnode),
-            pReq->subKey);
+    tqError("tmq poll: no consumer handle for consumer:%" PRId64 ", in vgId:%d, subkey %s", consumerId,
+            TD_VID(pTq->pVnode), pReq->subKey);
     return -1;
   }
 
   // check rebalance
   if (pHandle->consumerId != consumerId) {
-    tqError("tmq poll: consumer handle mismatch for consumer %ld in vg %d, subkey %s, handle consumer id %ld",
+    tqError("tmq poll: consumer handle mismatch for consumer:%" PRId64
+            ", in vgId:%d, subkey %s, handle consumer id %" PRId64,
             consumerId, TD_VID(pTq->pVnode), pReq->subKey, pHandle->consumerId);
     return -1;
   }
@@ -270,6 +267,14 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     consumerEpoch = atomic_val_compare_exchange_32(&pHandle->epoch, consumerEpoch, reqEpoch);
   }
 
+  char buf[80];
+  tFormatOffset(buf, 80, &reqOffset);
+  tqDebug("tmq poll: consumer %ld (epoch %d), subkey %s, recv poll req in vg %d, req offset %s", consumerId,
+          pReq->epoch, pHandle->subKey, TD_VID(pTq->pVnode), buf);
+
+  SMqDataRsp dataRsp = {0};
+  tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
+
   // 2.reset offset if needed
   if (reqOffset.type > 0) {
     fetchOffsetNew = reqOffset;
@@ -279,7 +284,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       fetchOffsetNew = pOffset->val;
       char formatBuf[80];
       tFormatOffset(formatBuf, 80, &fetchOffsetNew);
-      tqDebug("tmq poll: consumer %ld, offset reset to %s", consumerId, formatBuf);
+      tqDebug("tmq poll: consumer %" PRId64 ", subkey %s, offset reset to %s", consumerId, pHandle->subKey, formatBuf);
     } else {
       if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) {
         if (pReq->useSnapshot && pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
@@ -293,25 +298,50 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
           tqOffsetResetToLog(&fetchOffsetNew, walGetFirstVer(pTq->pVnode->pWal));
         }
       } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
-        tqOffsetResetToLog(&fetchOffsetNew, walGetLastVer(pTq->pVnode->pWal));
+        tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
+        tqDebug("tmq poll: consumer %ld, subkey %s, offset reset to %ld", consumerId, pHandle->subKey,
+                dataRsp.rspOffset.version);
+        if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
+          code = -1;
+        }
+        goto OVER;
       } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) {
-        tqError("tmq poll: no offset committed for consumer %ld in vg %d, subkey %s, reset none failed", consumerId,
-                TD_VID(pTq->pVnode), pReq->subKey);
+        tqError("tmq poll: subkey %s, no offset committed for consumer %" PRId64
+                " in vg %d, subkey %s, reset none failed",
+                pHandle->subKey, consumerId, TD_VID(pTq->pVnode), pReq->subKey);
         terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
-        return -1;
+        code = -1;
+        goto OVER;
       }
     }
   }
 
   // 3.query
-  SMqDataRsp dataRsp = {0};
-  tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
+  if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
+    if (fetchOffsetNew.type == TMQ_OFFSET__LOG) {
+      fetchOffsetNew.version++;
+    }
+    if (tqScan(pTq, &pHandle->execHandle, &dataRsp, &fetchOffsetNew) < 0) {
+      ASSERT(0);
+      code = -1;
+      goto OVER;
+    }
+    if (dataRsp.blockNum == 0) {
+      // TODO add to async task pool
+      /*dataRsp.rspOffset.version--;*/
+    }
+    if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
+      code = -1;
+    }
+    goto OVER;
+  }
 
-  if (fetchOffsetNew.type == TMQ_OFFSET__LOG) {
+  if (pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN) {
     int64_t     fetchVer = fetchOffsetNew.version + 1;
     SWalCkHead* pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048);
     if (pCkHead == NULL) {
-      return -1;
+      code = -1;
+      goto OVER;
     }
 
     walSetReaderCapacity(pHandle->pWalReader, 2048);
@@ -319,8 +349,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     while (1) {
       consumerEpoch = atomic_load_32(&pHandle->epoch);
       if (consumerEpoch > reqEpoch) {
-        tqWarn("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d, discard req epoch %d",
-               consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchVer, consumerEpoch, reqEpoch);
+        tqWarn("tmq poll: consumer %ld (epoch %d), subkey %s, vg %d offset %" PRId64
+               ", found new consumer epoch %d, discard req epoch %d",
+               consumerId, pReq->epoch, pHandle->subKey, TD_VID(pTq->pVnode), fetchVer, consumerEpoch, reqEpoch);
         break;
       }
 
@@ -337,8 +368,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 
       SWalCont* pHead = &pCkHead->head;
 
-      tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
-              TD_VID(pTq->pVnode), fetchVer, pHead->msgType);
+      tqDebug("tmq poll: consumer:%" PRId64 ", (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", consumerId,
+              pReq->epoch, TD_VID(pTq->pVnode), fetchVer, pHead->msgType);
 
       if (pHead->msgType == TDMT_VND_SUBMIT) {
         SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
@@ -363,7 +394,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       } else {
         ASSERT(pHandle->fetchMeta);
         ASSERT(IS_META_MSG(pHead->msgType));
-        tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType);
+        tqInfo("fetch meta msg, ver:%" PRId64 ", type:%d", pHead->version, pHead->msgType);
         SMqMetaRsp metaRsp = {0};
         /*metaRsp.reqOffset = pReq->reqOffset.version;*/
         /*metaRsp.rspOffset = fetchVer;*/
@@ -383,8 +414,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     }
 
     taosMemoryFree(pCkHead);
+#if 0
   } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_DATA) {
-    tqInfo("retrieve using snapshot actual offset: uid %ld ts %ld", fetchOffsetNew.uid, fetchOffsetNew.ts);
+    tqInfo("retrieve using snapshot actual offset: uid %" PRId64 " ts %" PRId64, fetchOffsetNew.uid, fetchOffsetNew.ts);
     if (tqScanSnapshot(pTq, &pHandle->execHandle, &dataRsp, fetchOffsetNew, workerId) < 0) {
       ASSERT(0);
     }
@@ -393,6 +425,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
       code = -1;
     }
+#endif
   } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_META) {
     ASSERT(0);
   }
@@ -482,10 +515,10 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
       pHandle->execHandle.execTb.suid = req.suid;
       SArray* tbUidList = taosArrayInit(0, sizeof(int64_t));
       vnodeGetCtbIdList(pTq->pVnode, req.suid, tbUidList);
-      tqDebug("vg %d, tq try get suid: %ld", pTq->pVnode->config.vgId, req.suid);
+      tqDebug("vgId:%d, tq try get suid:%" PRId64, pTq->pVnode->config.vgId, req.suid);
       for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) {
         int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i);
-        tqDebug("vg %d, idx %d, uid: %ld", TD_VID(pTq->pVnode), i, tbUid);
+        tqDebug("vgId:%d, idx %d, uid:%" PRId64, TD_VID(pTq->pVnode), i, tbUid);
       }
       for (int32_t i = 0; i < 5; i++) {
         tqReaderSetTbUidList(pHandle->execHandle.pExecReader[i], tbUidList);
@@ -572,7 +605,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) {
 
   streamSetupTrigger(pTask);
 
-  tqInfo("deploy stream task id %d child id %d on vg %d", pTask->taskId, pTask->selfChildId, TD_VID(pTq->pVnode));
+  tqInfo("deploy stream task id %d child id %d on vgId:%d", pTask->taskId, pTask->selfChildId, TD_VID(pTq->pVnode));
 
   taosHashPut(pTq->pStreamTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*));
 

source/dnode/vnode/src/tq/tqExec.c

@@ -46,7 +46,7 @@ static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, int32_t workerI
   return 0;
 }
 
-static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, int32_t workerId) {
+static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp) {
   SMetaReader mr = {0};
   metaReaderInit(&mr, pTq->pVnode->pMeta, 0);
   if (metaGetTableEntryByUid(&mr, uid) < 0) {
@@ -59,15 +59,70 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, i
   return 0;
 }
 
+int64_t tqScan(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* pOffset) {
+  qTaskInfo_t task = pExec->execCol.task[0];
+
+  if (qStreamPrepareScan(task, pOffset) < 0) {
+    ASSERT(pOffset->type == TMQ_OFFSET__LOG);
+    pRsp->rspOffset = *pOffset;
+    pRsp->rspOffset.version--;
+    return 0;
+  }
+
+  int32_t rowCnt = 0;
+  while (1) {
+    SSDataBlock* pDataBlock = NULL;
+    uint64_t     ts = 0;
+    if (qExecTask(task, &pDataBlock, &ts) < 0) {
+      ASSERT(0);
+    }
+
+    if (pDataBlock != NULL) {
+      tqAddBlockDataToRsp(pDataBlock, pRsp);
+      pRsp->blockNum++;
+      if (pRsp->withTbName) {
+        if (pOffset->type == TMQ_OFFSET__LOG) {
+          int64_t uid = pExec->pExecReader[0]->msgIter.uid;
+          tqAddTbNameToRsp(pTq, uid, pRsp);
+        } else {
+          pRsp->withTbName = 0;
+        }
+      }
+      if (pOffset->type == TMQ_OFFSET__LOG) {
+        continue;
+      } else {
+        rowCnt += pDataBlock->info.rows;
+        if (rowCnt <= 4096) continue;
+      }
+    }
+
+    void* meta = qStreamExtractMetaMsg(task);
+    if (meta != NULL) {
+      // tq add meta to rsp
+    }
+
+    if (qStreamExtractOffset(task, &pRsp->rspOffset) < 0) {
+      ASSERT(0);
+    }
+
+    ASSERT(pRsp->rspOffset.type != 0);
+
+    if (pRsp->rspOffset.type == TMQ_OFFSET__LOG) {
+      ASSERT(pRsp->rspOffset.version + 1 >= pRsp->reqOffset.version);
+    }
+
+    break;
+  }
+
+  return 0;
+}
+
+#if 0
 int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal offset, int32_t workerId) {
   ASSERT(pExec->subType == TOPIC_SUB_TYPE__COLUMN);
   qTaskInfo_t task = pExec->execCol.task[workerId];
 
-  /*if (qStreamScanSnapshot(task) < 0) {*/
-  /*ASSERT(0);*/
-  /*}*/
-
-  if (qStreamPrepareScan(task, offset.uid, offset.ts) < 0) {
+  if (qStreamPrepareTsdbScan(task, offset.uid, offset.ts) < 0) {
     ASSERT(0);
   }
 
@@ -93,7 +148,7 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, S
       if (qGetStreamScanStatus(task, &uid, &ts) < 0) {
         ASSERT(0);
       }
-      tqAddTbNameToRsp(pTq, uid, pRsp, workerId);
+      tqAddTbNameToRsp(pTq, uid, pRsp);
 #endif
     }
     pRsp->blockNum++;
@@ -110,6 +165,7 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, S
 
   return 0;
 }
+#endif
 
 int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataRsp* pRsp, int32_t workerId) {
   if (pExec->subType == TOPIC_SUB_TYPE__COLUMN) {
@@ -129,7 +185,7 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR
       tqAddBlockDataToRsp(pDataBlock, pRsp);
       if (pRsp->withTbName) {
         int64_t uid = pExec->pExecReader[workerId]->msgIter.uid;
-        tqAddTbNameToRsp(pTq, uid, pRsp, workerId);
+        tqAddTbNameToRsp(pTq, uid, pRsp);
       }
       pRsp->blockNum++;
     }
@@ -146,7 +202,7 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR
       tqAddBlockDataToRsp(&block, pRsp);
       if (pRsp->withTbName) {
         int64_t uid = pExec->pExecReader[workerId]->msgIter.uid;
-        tqAddTbNameToRsp(pTq, uid, pRsp, workerId);
+        tqAddTbNameToRsp(pTq, uid, pRsp);
       }
       tqAddBlockSchemaToRsp(pExec, workerId, pRsp);
       pRsp->blockNum++;
@@ -164,7 +220,7 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR
       tqAddBlockDataToRsp(&block, pRsp);
       if (pRsp->withTbName) {
         int64_t uid = pExec->pExecReader[workerId]->msgIter.uid;
-        tqAddTbNameToRsp(pTq, uid, pRsp, workerId);
+        tqAddTbNameToRsp(pTq, uid, pRsp);
       }
       tqAddBlockSchemaToRsp(pExec, workerId, pRsp);
       pRsp->blockNum++;

source/dnode/vnode/src/tq/tqOffset.c

@@ -139,7 +139,7 @@ int32_t tqOffsetSnapshot(STqOffsetStore* pStore) {
     int64_t writeLen;
     if ((writeLen = taosWriteFile(pFile, buf, totLen)) != totLen) {
       ASSERT(0);
-      tqError("write offset incomplete, len %d, write len %ld", bodyLen, writeLen);
+      tqError("write offset incomplete, len %d, write len %" PRId64, bodyLen, writeLen);
       taosHashCancelIterate(pStore->pHash, pIter);
       return -1;
     }

source/dnode/vnode/src/tq/tqPush.c

@@ -223,7 +223,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
     memset(&pHandle->pushHandle.rpcInfo, 0, sizeof(SRpcHandleInfo));
     taosWUnLockLatch(&pHandle->pushHandle.lock);
 
-    tqDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
+    tqDebug("vgId:%d offset %" PRId64 " from consumer:%" PRId64 ", (epoch %d) send rsp, block num: %d, reqOffset:%" PRId64 ", rspOffset:%" PRId64,
             TD_VID(pTq->pVnode), fetchOffset, pHandle->pushHandle.consumerId, pHandle->pushHandle.epoch, rsp.blockNum,
             rsp.reqOffset, rsp.rspOffset);
 

source/dnode/vnode/src/tq/tqRead.c

@@ -15,11 +15,6 @@
 
 #include "tq.h"
 
-int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal offset) {
-  /*if ()*/
-  return 0;
-}
-
 int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** ppCkHead) {
   int32_t code = 0;
   taosThreadMutexLock(&pHandle->pWalReader->mutex);
@@ -27,8 +22,8 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea
 
   while (1) {
     if (walFetchHead(pHandle->pWalReader, offset, *ppCkHead) < 0) {
-      tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", pHandle->consumerId,
-              pHandle->epoch, TD_VID(pTq->pVnode), offset);
+      tqDebug("tmq poll: consumer:%" PRId64 ", (epoch %d) vgId:%d offset %" PRId64 ", no more log to return",
+              pHandle->consumerId, pHandle->epoch, TD_VID(pTq->pVnode), offset);
       *fetchOffset = offset - 1;
       code = -1;
       goto END;
@@ -84,8 +79,10 @@ STqReader* tqOpenReader(SVnode* pVnode) {
     return NULL;
   }
 
-  // TODO open
-  /*pReader->pWalReader = walOpenReader(pVnode->pWal, NULL);*/
+  pReader->pWalReader = walOpenReader(pVnode->pWal, NULL);
+  if (pReader->pWalReader == NULL) {
+    return NULL;
+  }
 
   pReader->pVnodeMeta = pVnode->pMeta;
   pReader->pMsg = NULL;
@@ -106,13 +103,27 @@ void tqCloseReader(STqReader* pReader) {
   taosMemoryFree(pReader);
 }
 
+int32_t tqSeekVer(STqReader* pReader, int64_t ver) {
+  if (walReadSeekVer(pReader->pWalReader, ver) < 0) {
+    ASSERT(pReader->pWalReader->curInvalid);
+    ASSERT(pReader->pWalReader->curVersion == ver);
+    return -1;
+  }
+  ASSERT(pReader->pWalReader->curVersion == ver);
+  return 0;
+}
+
 int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
   bool fromProcessedMsg = pReader->pMsg != NULL;
 
   while (1) {
     if (!fromProcessedMsg) {
       if (walNextValidMsg(pReader->pWalReader) < 0) {
+        pReader->ver = pReader->pWalReader->curVersion - pReader->pWalReader->curInvalid;
+        ret->offset.type = TMQ_OFFSET__LOG;
+        ret->offset.version = pReader->ver;
         ret->fetchType = FETCH_TYPE__NONE;
+        ASSERT(ret->offset.version >= 0);
         return -1;
       }
       void* body = pReader->pWalReader->pHead->head.body;
@@ -127,22 +138,21 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
     }
 
     while (tqNextDataBlock(pReader)) {
+      // TODO mem free
       memset(&ret->data, 0, sizeof(SSDataBlock));
       int32_t code = tqRetrieveDataBlock(&ret->data, pReader);
       if (code != 0 || ret->data.info.rows == 0) {
-        if (fromProcessedMsg) {
-          ret->fetchType = FETCH_TYPE__NONE;
-          return 0;
-        } else {
-          break;
-        }
+        ASSERT(0);
+        continue;
       }
-
       ret->fetchType = FETCH_TYPE__DATA;
       return 0;
     }
 
     if (fromProcessedMsg) {
+      ret->offset.type = TMQ_OFFSET__LOG;
+      ret->offset.version = pReader->ver;
+      ASSERT(pReader->ver >= 0);
       ret->fetchType = FETCH_TYPE__NONE;
       return 0;
     }
@@ -179,9 +189,9 @@ bool tqNextDataBlock(STqReader* pReader) {
       return true;
     }
     void* ret = taosHashGet(pReader->tbIdHash, &pReader->msgIter.uid, sizeof(int64_t));
-    /*tqDebug("search uid %ld", pHandle->msgIter.uid);*/
+    /*tqDebug("search uid %" PRId64, pHandle->msgIter.uid);*/
     if (ret != NULL) {
-      /*tqDebug("find   uid %ld", pHandle->msgIter.uid);*/
+      /*tqDebug("find   uid %" PRId64, pHandle->msgIter.uid);*/
       return true;
     }
   }
@@ -212,7 +222,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) {
     if (pReader->pSchema) taosMemoryFree(pReader->pSchema);
     pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion);
     if (pReader->pSchema == NULL) {
-      tqWarn("cannot found tsschema for table: uid: %ld (suid: %ld), version %d, possibly dropped table",
+      tqWarn("cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 "), version %d, possibly dropped table",
              pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer);
       /*ASSERT(0);*/
       terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
@@ -222,7 +232,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) {
     if (pReader->pSchemaWrapper) tDeleteSSchemaWrapper(pReader->pSchemaWrapper);
     pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, true);
     if (pReader->pSchemaWrapper == NULL) {
-      tqWarn("cannot found schema wrapper for table: suid: %ld, version %d, possibly dropped table",
+      tqWarn("cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table",
              pReader->msgIter.uid, pReader->cachedSchemaVer);
       /*ASSERT(0);*/
       terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -2182,12 +2182,21 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret
   return VND_TSDB(pVnode);
 }
 
-static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level) {
+SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level) {
+  int64_t startVer = (pCond->startVersion == -1)? 0:pCond->startVersion;
+
   if (VND_IS_RSMA(pVnode)) {
-    return (SVersionRange){.minVer = pCond->startVersion, .maxVer = tdRSmaGetMaxSubmitVer(pVnode->pSma, level)};
+    return (SVersionRange){.minVer = startVer, .maxVer = tdRSmaGetMaxSubmitVer(pVnode->pSma, level)};
+  }
+
+  int64_t endVer = 0;
+  if (pCond->endVersion == -1) {  // user not specified end version, set current maximum version of vnode as the endVersion
+    endVer = pVnode->state.applied;
+  } else {
+    endVer = (pCond->endVersion > pVnode->state.applied)? pVnode->state.applied:pCond->endVersion;
   }
 
-  return (SVersionRange){.minVer = pCond->startVersion, .maxVer = pVnode->state.applied};
+  return (SVersionRange){.minVer = startVer, .maxVer = endVer};
 }
 
 // // todo not unref yet, since it is not support multi-group interpolation query

source/dnode/vnode/src/vnd/vnodeUtil.c → source/dnode/vnode/src/tsdb/tsdbRetention.c

@@ -13,33 +13,30 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnd.h"
+#include "tsdb.h"
 
-int32_t vnodeRealloc(void** pp, int32_t size) {
-  uint8_t* p = NULL;
-  int32_t  csize = 0;
+int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) {
+  int32_t code = 0;
 
-  if (*pp) {
-    p = (uint8_t*)(*pp) - sizeof(int32_t);
-    csize = *(int32_t*)p;
-  }
+  // begin
+  code = tsdbFSBegin(pTsdb->fs);
+  if (code) goto _err;
 
-  if (csize >= size) {
-    return 0;
-  }
+  // do retention
+  for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs->nState->aDFileSet); iSet++) {
+    SDFileSet *pDFileSet = (SDFileSet *)taosArrayGet(pTsdb->fs->nState->aDFileSet, iSet);
 
-  p = (uint8_t*)taosMemoryRealloc(p, size);
-  if (p == NULL) {
-    return TSDB_CODE_OUT_OF_MEMORY;
+    // TODO
   }
-  *(int32_t*)p = size;
-  *pp = p + sizeof(int32_t);
 
-  return 0;
-}
+  // commit
+  code = tsdbFSCommit(pTsdb->fs);
+  if (code) goto _err;
 
-void vnodeFree(void* p) {
-  if (p) {
-    taosMemoryFree(((uint8_t*)p) - sizeof(int32_t));
-  }
+_exit:
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb do retention failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  return code;
 }
\ No newline at end of file

source/dnode/vnode/src/tsdb/tsdbUtil.c

@@ -87,8 +87,10 @@ int32_t tPutMapData(uint8_t *p, SMapData *pMapData) {
 
   n += tPutI32v(p ? p + n : p, pMapData->nItem);
   if (pMapData->nItem) {
+    int32_t lOffset = 0;
     for (int32_t iItem = 0; iItem < pMapData->nItem; iItem++) {
-      n += tPutI32v(p ? p + n : p, pMapData->aOffset[iItem]);
+      n += tPutI32v(p ? p + n : p, pMapData->aOffset[iItem] - lOffset);
+      lOffset = pMapData->aOffset[iItem];
     }
 
     n += tPutI32v(p ? p + n : p, pMapData->nData);
@@ -111,8 +113,11 @@ int32_t tGetMapData(uint8_t *p, SMapData *pMapData) {
   if (pMapData->nItem) {
     if (tRealloc((uint8_t **)&pMapData->aOffset, sizeof(int32_t) * pMapData->nItem)) return -1;
 
+    int32_t lOffset = 0;
     for (int32_t iItem = 0; iItem < pMapData->nItem; iItem++) {
       n += tGetI32v(p + n, &pMapData->aOffset[iItem]);
+      pMapData->aOffset[iItem] += lOffset;
+      lOffset = pMapData->aOffset[iItem];
     }
 
     n += tGetI32v(p + n, &pMapData->nData);

source/dnode/vnode/src/vnd/vnodeCommit.c

@@ -223,6 +223,7 @@ int vnodeCommit(SVnode *pVnode) {
   // save info
   info.config = pVnode->config;
   info.state.committed = pVnode->state.applied;
+  info.state.commitTerm = pVnode->state.applyTerm;
   info.state.commitID = pVnode->state.commitID;
   snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path);
   if (vnodeSaveInfo(dir, &info) < 0) {
@@ -270,7 +271,7 @@ int vnodeCommit(SVnode *pVnode) {
     ASSERT(0);
     return -1;
   }
-  
+
   pVnode->state.committed = info.state.committed;
 
   // postCommit
@@ -316,6 +317,7 @@ static int vnodeEncodeState(const void *pObj, SJson *pJson) {
 
   if (tjsonAddIntegerToObject(pJson, "commit version", pState->committed) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "commit ID", pState->commitID) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "commit term", pState->commitTerm) < 0) return -1;
 
   return 0;
 }
@@ -328,6 +330,8 @@ static int vnodeDecodeState(const SJson *pJson, void *pObj) {
   if (code < 0) return -1;
   tjsonGetNumberValue(pJson, "commit ID", pState->commitID, code);
   if (code < 0) return -1;
+  tjsonGetNumberValue(pJson, "commit term", pState->commitTerm, code);
+  if (code < 0) return -1;
 
   return 0;
 }

source/dnode/vnode/src/vnd/vnodeModule.c

@@ -100,6 +100,7 @@ void vnodeCleanup() {
 
   walCleanUp();
   tqCleanUp();
+  smaCleanUp();
 }
 
 int vnodeScheduleTask(int (*execute)(void*), void* arg) {

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -79,8 +79,10 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
   strcpy(pVnode->path, path);
   pVnode->config = info.config;
   pVnode->state.committed = info.state.committed;
+  pVnode->state.commitTerm = info.state.commitTerm;
   pVnode->state.applied = info.state.committed;
   pVnode->state.commitID = info.state.commitID;
+  pVnode->state.commitTerm = info.state.commitTerm;
   pVnode->pTfs = pTfs;
   pVnode->msgCb = msgCb;
   pVnode->blockCount = 0;
@@ -194,4 +196,9 @@ void vnodeStop(SVnode *pVnode) {}
 
 int64_t vnodeGetSyncHandle(SVnode *pVnode) { return pVnode->sync; }
 
-void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) { pSnapshot->lastApplyIndex = pVnode->state.committed; }
+void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) {
+  pSnapshot->data = NULL;
+  pSnapshot->lastApplyIndex = pVnode->state.committed;
+  pSnapshot->lastApplyTerm = pVnode->state.commitTerm;
+  pSnapshot->lastConfigIndex = -1;
+}

source/dnode/vnode/src/vnd/vnodeSnapshot.c

@@ -13,24 +13,27 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
-
-struct SVSnapshotReader {
-  SVnode              *pVnode;
-  int64_t              sver;
-  int64_t              ever;
-  int8_t               isMetaEnd;
-  int8_t               isTsdbEnd;
-  SMetaSnapshotReader *pMetaReader;
-  STsdbSnapshotReader *pTsdbReader;
-  void                *pData;
-  int32_t              nData;
+#include "vnd.h"
+
+// SVSnapReader ========================================================
+struct SVSnapReader {
+  SVnode *pVnode;
+  int64_t sver;
+  int64_t ever;
+  // meta
+  int8_t           metaDone;
+  SMetaSnapReader *pMetaReader;
+  // tsdb
+  int8_t           tsdbDone;
+  STsdbSnapReader *pTsdbReader;
+  uint8_t         *pData;
 };
 
-int32_t vnodeSnapshotReaderOpen(SVnode *pVnode, SVSnapshotReader **ppReader, int64_t sver, int64_t ever) {
-  SVSnapshotReader *pReader = NULL;
+int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapReader **ppReader) {
+  int32_t       code = 0;
+  SVSnapReader *pReader = NULL;
 
-  pReader = (SVSnapshotReader *)taosMemoryCalloc(1, sizeof(*pReader));
+  pReader = (SVSnapReader *)taosMemoryCalloc(1, sizeof(*pReader));
   if (pReader == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
@@ -38,72 +41,169 @@ int32_t vnodeSnapshotReaderOpen(SVnode *pVnode, SVSnapshotReader **ppReader, int
   pReader->pVnode = pVnode;
   pReader->sver = sver;
   pReader->ever = ever;
-  pReader->isMetaEnd = 0;
-  pReader->isTsdbEnd = 0;
 
-  if (metaSnapshotReaderOpen(pVnode->pMeta, &pReader->pMetaReader, sver, ever) < 0) {
-    taosMemoryFree(pReader);
-    goto _err;
-  }
+  code = metaSnapReaderOpen(pVnode->pMeta, sver, ever, &pReader->pMetaReader);
+  if (code) goto _err;
 
-  if (tsdbSnapshotReaderOpen(pVnode->pTsdb, &pReader->pTsdbReader, sver, ever) < 0) {
-    metaSnapshotReaderClose(pReader->pMetaReader);
-    taosMemoryFree(pReader);
-    goto _err;
-  }
+  code = tsdbSnapReaderOpen(pVnode->pTsdb, sver, ever, &pReader->pTsdbReader);
+  if (code) goto _err;
 
-_exit:
   *ppReader = pReader;
-  return 0;
+  return code;
 
 _err:
+  vError("vgId:%d vnode snapshot reader open failed since %s", TD_VID(pVnode), tstrerror(code));
   *ppReader = NULL;
-  return -1;
+  return code;
 }
 
-int32_t vnodeSnapshotReaderClose(SVSnapshotReader *pReader) {
-  if (pReader) {
-    vnodeFree(pReader->pData);
-    tsdbSnapshotReaderClose(pReader->pTsdbReader);
-    metaSnapshotReaderClose(pReader->pMetaReader);
-    taosMemoryFree(pReader);
-  }
-  return 0;
+int32_t vnodeSnapReaderClose(SVSnapReader *pReader) {
+  int32_t code = 0;
+
+  tFree(pReader->pData);
+  if (pReader->pTsdbReader) tsdbSnapReaderClose(&pReader->pTsdbReader);
+  if (pReader->pMetaReader) metaSnapReaderClose(&pReader->pMetaReader);
+  taosMemoryFree(pReader);
+
+  return code;
 }
 
-int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32_t *nData) {
+int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) {
   int32_t code = 0;
 
-  if (!pReader->isMetaEnd) {
-    code = metaSnapshotRead(pReader->pMetaReader, &pReader->pData, &pReader->nData);
+  if (!pReader->metaDone) {
+    code = metaSnapRead(pReader->pMetaReader, &pReader->pData);
     if (code) {
       if (code == TSDB_CODE_VND_READ_END) {
-        pReader->isMetaEnd = 1;
+        pReader->metaDone = 1;
       } else {
-        return code;
+        goto _err;
       }
     } else {
       *ppData = pReader->pData;
-      *nData = pReader->nData;
-      return code;
+      *nData = sizeof(SSnapDataHdr) + ((SSnapDataHdr *)pReader->pData)->size;
+      goto _exit;
     }
   }
 
-  if (!pReader->isTsdbEnd) {
-    code = tsdbSnapshotRead(pReader->pTsdbReader, &pReader->pData, &pReader->nData);
+  if (!pReader->tsdbDone) {
+    code = tsdbSnapRead(pReader->pTsdbReader, &pReader->pData);
     if (code) {
       if (code == TSDB_CODE_VND_READ_END) {
-        pReader->isTsdbEnd = 1;
+        pReader->tsdbDone = 1;
       } else {
-        return code;
+        goto _err;
       }
     } else {
       *ppData = pReader->pData;
-      *nData = pReader->nData;
-      return code;
+      *nData = sizeof(SSnapDataHdr) + ((SSnapDataHdr *)pReader->pData)->size;
+      goto _exit;
     }
   }
 
   code = TSDB_CODE_VND_READ_END;
+
+_exit:
+  return code;
+
+_err:
+  vError("vgId:% snapshot read failed since %s", TD_VID(pReader->pVnode), tstrerror(code));
+  return code;
+}
+
+// SVSnapWriter ========================================================
+struct SVSnapWriter {
+  SVnode *pVnode;
+  int64_t sver;
+  int64_t ever;
+  // meta
+  SMetaSnapWriter *pMetaSnapWriter;
+  // tsdb
+  STsdbSnapWriter *pTsdbSnapWriter;
+};
+
+static int32_t vnodeSnapRollback(SVSnapWriter *pWriter) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+static int32_t vnodeSnapCommit(SVSnapWriter *pWriter) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter) {
+  int32_t       code = 0;
+  SVSnapWriter *pWriter = NULL;
+
+  // alloc
+  pWriter = (SVSnapWriter *)taosMemoryCalloc(1, sizeof(*pWriter));
+  if (pWriter == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pWriter->pVnode = pVnode;
+  pWriter->sver = sver;
+  pWriter->ever = ever;
+
+  return code;
+
+_err:
+  vError("vgId:%d vnode snapshot writer open failed since %s", TD_VID(pVnode), tstrerror(code));
+  return code;
+}
+
+int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback) {
+  int32_t code = 0;
+
+  if (rollback) {
+    code = vnodeSnapRollback(pWriter);
+    if (code) goto _err;
+  } else {
+    code = vnodeSnapCommit(pWriter);
+    if (code) goto _err;
+  }
+
+  taosMemoryFree(pWriter);
+  return code;
+
+_err:
+  vError("vgId:%d vnode snapshow writer close failed since %s", TD_VID(pWriter->pVnode), tstrerror(code));
+  return code;
+}
+
+int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
+  int32_t       code = 0;
+  SSnapDataHdr *pSnapDataHdr = (SSnapDataHdr *)pData;
+  SVnode       *pVnode = pWriter->pVnode;
+
+  ASSERT(pSnapDataHdr->size + sizeof(SSnapDataHdr) == nData);
+
+  if (pSnapDataHdr->type == 0) {
+    // meta
+    if (pWriter->pMetaSnapWriter == NULL) {
+      code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter);
+      if (code) goto _err;
+    }
+
+    code = metaSnapWrite(pWriter->pMetaSnapWriter, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
+    if (code) goto _err;
+  } else {
+    // tsdb
+    if (pWriter->pTsdbSnapWriter == NULL) {
+      code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, &pWriter->pTsdbSnapWriter);
+      if (code) goto _err;
+    }
+
+    code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
+    if (code) goto _err;
+  }
+
+  return code;
+
+_err:
+  vError("vgId:%d vnode snapshot write failed since %s", TD_VID(pVnode), tstrerror(code));
   return code;
 }
\ No newline at end of file

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -143,6 +143,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
          version);
 
   pVnode->state.applied = version;
+  pVnode->state.applyTerm = pMsg->info.conn.applyTerm;
 
   // skip header
   pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));

source/libs/catalog/inc/catalogInt.h

@@ -166,7 +166,7 @@ typedef struct SCtgDBCache {
   int8_t           deleted;
   SCtgVgCache      vgCache;
   SHashObj        *tbCache;         // key:tbname, value:SCtgTbCache
-  SHashObj        *stbCache;        // key:suid, value:STableMeta*
+  SHashObj        *stbCache;        // key:suid, value:char*
 } SCtgDBCache;
 
 typedef struct SCtgRentSlot {

source/libs/catalog/src/ctgDbg.c

@@ -64,7 +64,7 @@ void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) {
       qDebug("db %d vgInfo:", i);
       for (int32_t j = 0; j < vgNum; ++j) {
         SVgroupInfo* pInfo = taosArrayGet(pDb, j);
-        qDebug("vg %d info: vgId:%d", j, pInfo->vgId);
+        qDebug("vg :%d info: vgId:%d", j, pInfo->vgId);
       }
     }
   } else {

source/libs/catalog/src/ctgUtil.c

@@ -731,7 +731,7 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName
 
   *pVgroup = *vgInfo;
 
-  ctgDebug("Got tb %s hash vgroup, vgId %d, epNum %d, current %s port %d", tbFullName, vgInfo->vgId, vgInfo->epSet.numOfEps,
+  ctgDebug("Got tb %s hash vgroup, vgId:%d, epNum %d, current %s port %d", tbFullName, vgInfo->vgId, vgInfo->epSet.numOfEps,
     vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port);
 
   CTG_RET(code);

source/libs/executor/src/executor.c

@@ -60,9 +60,9 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
         taosArrayAddAll(p->pDataBlock, pDataBlock->pDataBlock);
         taosArrayPush(pInfo->pBlockLists, &p);
       }
-    } else if (type == STREAM_INPUT__DATA_SCAN) {
+    } else if (type == STREAM_INPUT__TABLE_SCAN) {
       // do nothing
-      ASSERT(pInfo->blockType == STREAM_INPUT__DATA_SCAN);
+      ASSERT(pInfo->blockType == STREAM_INPUT__TABLE_SCAN);
     } else {
       ASSERT(0);
     }
@@ -76,7 +76,7 @@ int32_t qStreamScanSnapshot(qTaskInfo_t tinfo) {
     return TSDB_CODE_QRY_APP_ERROR;
   }
   SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
-  return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_INPUT__DATA_SCAN, 0, NULL);
+  return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_INPUT__TABLE_SCAN, 0, NULL);
 }
 
 int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type, bool assignUid) {

source/libs/function/inc/fnLog.h

@@ -14,7 +14,7 @@ extern "C" {
 #define fnFatal(...) { if (udfDebugFlag & DEBUG_FATAL) { taosPrintLog("UDF FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
 #define fnError(...) { if (udfDebugFlag & DEBUG_ERROR) { taosPrintLog("UDF ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
 #define fnWarn(...)  { if (udfDebugFlag & DEBUG_WARN)  { taosPrintLog("UDF WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
-#define fnInfo(...)  { if (udfDebugFlag & DEBUG_INFO)  { taosPrintLog("UDF  ", DEBUG_INFO, 255, __VA_ARGS__); }}
+#define fnInfo(...)  { if (udfDebugFlag & DEBUG_INFO)  { taosPrintLog("UDF ", DEBUG_INFO, 255, __VA_ARGS__); }}
 #define fnDebug(...) { if (udfDebugFlag & DEBUG_DEBUG) { taosPrintLog("UDF ", DEBUG_DEBUG, udfDebugFlag, __VA_ARGS__); }}
 #define fnTrace(...) { if (udfDebugFlag & DEBUG_TRACE) { taosPrintLog("UDF ", DEBUG_TRACE, udfDebugFlag, __VA_ARGS__); }}
 // clang-format on