Merge branch '3.0' into feature/qnode

docs-en/14-reference/03-connector/rust.mdx

@@ -45,15 +45,15 @@ Add the [libtaos][libtaos] dependency to the [Rust](https://rust-lang.org) proje
 <Tabs defaultValue="native">
 <TabItem value="native" label="native connection">
 
-Add [libtaos][libtaos] to the ``Cargo.toml`'' file.
+Add [libtaos][libtaos] to the `Cargo.toml` file.
 
-``toml
+```toml
 [dependencies]
 # use default feature
 libtaos = "*"
 ```
 
-</TabItem
+</TabItem>
 <TabItem value="rest" label="REST connection">
 
 Add [libtaos][libtaos] to the `Cargo.toml` file and enable the `rest` feature.

include/common/tcommon.h

@@ -50,6 +50,7 @@ typedef enum EStreamType {
   STREAM_INVERT,
   STREAM_REPROCESS,
   STREAM_INVALID,
+  STREAM_GET_ALL,
 } EStreamType;
 
 typedef struct {

include/common/tmsg.h

@@ -1496,6 +1496,7 @@ typedef struct {
 
 #define STREAM_TRIGGER_AT_ONCE          1
 #define STREAM_TRIGGER_WINDOW_CLOSE     2
+#define STREAM_TRIGGER_MAX_DELAY        3
 
 typedef struct {
   char    name[TSDB_TABLE_FNAME_LEN];
@@ -2335,24 +2336,26 @@ typedef struct {
 } SVgEpSet;
 
 typedef struct {
-  int8_t    version;       // for compatibility(default 0)
-  int8_t    intervalUnit;  // MACRO: TIME_UNIT_XXX
-  int8_t    slidingUnit;   // MACRO: TIME_UNIT_XXX
-  int8_t    timezoneInt;   // sma data expired if timezone changes.
-  int32_t   dstVgId;
-  char      indexName[TSDB_INDEX_NAME_LEN];
-  int32_t   exprLen;
-  int32_t   tagsFilterLen;
-  int32_t   numOfVgroups;
-  int64_t   indexUid;
-  tb_uid_t  tableUid;  // super/child/common table uid
-  int64_t   interval;
-  int64_t   offset;  // use unit by precision of DB
-  int64_t   sliding;
-  char*     expr;  // sma expression
-  char*     tagsFilter;
-  SVgEpSet* pVgEpSet;
-} STSma;  // Time-range-wise SMA
+  int8_t         version;       // for compatibility(default 0)
+  int8_t         intervalUnit;  // MACRO: TIME_UNIT_XXX
+  int8_t         slidingUnit;   // MACRO: TIME_UNIT_XXX
+  int8_t         timezoneInt;   // sma data expired if timezone changes.
+  int32_t        dstVgId;
+  char           indexName[TSDB_INDEX_NAME_LEN];
+  int32_t        exprLen;
+  int32_t        tagsFilterLen;
+  int64_t        indexUid;
+  tb_uid_t       tableUid;  // super/child/common table uid
+  tb_uid_t       dstTbUid;  // for dstVgroup
+  int64_t        interval;
+  int64_t        offset;  // use unit by precision of DB
+  int64_t        sliding;
+  char*          dstTbName;  // for dstVgroup
+  char*          expr;       // sma expression
+  char*          tagsFilter;
+  SSchemaWrapper schemaRow;  // for dstVgroup
+  SSchemaWrapper schemaTag;  // for dstVgroup
+} STSma;                     // Time-range-wise SMA
 
 typedef STSma SVCreateTSmaReq;
 
@@ -2436,27 +2439,6 @@ static int32_t tDecodeTSmaWrapper(SDecoder* pDecoder, STSmaWrapper* pReq) {
   return 0;
 }
 
-typedef struct {
-  int64_t     indexUid;
-  STimeWindow queryWindow;
-} SVGetTsmaExpWndsReq;
-
-#define SMA_WNDS_EXPIRE_FLAG      (0x1)
-#define SMA_WNDS_IS_EXPIRE(flag)  (((flag)&SMA_WNDS_EXPIRE_FLAG) != 0)
-#define SMA_WNDS_SET_EXPIRE(flag) ((flag) |= SMA_WNDS_EXPIRE_FLAG)
-
-typedef struct {
-  int64_t indexUid;
-  int8_t  flags;  // 0x1 all window expired
-  int32_t numExpWnds;
-  TSKEY   wndSKeys[];
-} SVGetTsmaExpWndsRsp;
-
-int32_t tEncodeSVGetTSmaExpWndsReq(SEncoder* pCoder, const SVGetTsmaExpWndsReq* pReq);
-int32_t tDecodeSVGetTsmaExpWndsReq(SDecoder* pCoder, SVGetTsmaExpWndsReq* pReq);
-int32_t tEncodeSVGetTSmaExpWndsRsp(SEncoder* pCoder, const SVGetTsmaExpWndsRsp* pReq);
-int32_t tDecodeSVGetTsmaExpWndsRsp(SDecoder* pCoder, SVGetTsmaExpWndsRsp* pReq);
-
 typedef struct {
   int idx;
 } SMCreateFullTextReq;
@@ -2516,7 +2498,8 @@ typedef struct {
 
 int32_t tSerializeSTableIndexRsp(void* buf, int32_t bufLen, const STableIndexRsp* pRsp);
 int32_t tDeserializeSTableIndexRsp(void* buf, int32_t bufLen, STableIndexRsp* pRsp);
-void    tFreeSTableIndexInfo(void* pInfo);
+
+void tFreeSTableIndexInfo(void* pInfo);
 
 typedef struct {
   int8_t  mqMsgType;

include/common/tmsgdef.h

@@ -190,7 +190,6 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT_RSMA, "vnode-submit-rsma", SSubmitReq, SSubmitRsp)
-  TD_DEF_MSG_TYPE(TDMT_VND_GET_TSMA_EXP_WNDS, "vnode-get-tsma-expired-windows", SVGetTsmaExpWndsReq, SVGetTsmaExpWndsRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "delete-data", SVDeleteReq, SVDeleteRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "alter-replica", NULL, NULL)

include/libs/parser/parser.h

@@ -64,7 +64,7 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata
 void qDestroyQuery(SQuery* pQueryNode);
 
 int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema);
-int32_t qSetSTableIdForRSma(SNode* pStmt, int64_t uid);
+int32_t qSetSTableIdForRsma(SNode* pStmt, int64_t uid);
 
 int32_t     qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash);
 int32_t     qResetStmtDataBlock(void* block, bool keepBuf);

include/util/taoserror.h

@@ -352,9 +352,6 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TDB_NO_CACHE_LAST_ROW         TAOS_DEF_ERROR_CODE(0, 0x0619)
 #define TSDB_CODE_TDB_TABLE_RECREATED           TAOS_DEF_ERROR_CODE(0, 0x061A)
 #define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR        TAOS_DEF_ERROR_CODE(0, 0x061B)
-#define TSDB_CODE_TDB_NO_SMA_INDEX_IN_META      TAOS_DEF_ERROR_CODE(0, 0x061C)
-#define TSDB_CODE_TDB_INVALID_SMA_STAT          TAOS_DEF_ERROR_CODE(0, 0x061D)
-#define TSDB_CODE_TDB_TSMA_ALREADY_EXIST        TAOS_DEF_ERROR_CODE(0, 0x061E)
 
 // query
 #define TSDB_CODE_QRY_INVALID_QHANDLE           TAOS_DEF_ERROR_CODE(0, 0x0700)
@@ -685,6 +682,19 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_SML_INVALID_DATA              TAOS_DEF_ERROR_CODE(0, 0x3002)
 #define TSDB_CODE_SML_INVALID_DB_CONF           TAOS_DEF_ERROR_CODE(0, 0x3003)
 
+//tsma
+#define TSDB_CODE_TSMA_ALREADY_EXIST             TAOS_DEF_ERROR_CODE(0, 0x3100)
+#define TSDB_CODE_TSMA_NO_INDEX_IN_META          TAOS_DEF_ERROR_CODE(0, 0x3101)
+#define TSDB_CODE_TSMA_INVALID_ENV               TAOS_DEF_ERROR_CODE(0, 0x3102)
+#define TSDB_CODE_TSMA_INVALID_STAT              TAOS_DEF_ERROR_CODE(0, 0x3103)
+#define TSDB_CODE_TSMA_NO_INDEX_IN_CACHE         TAOS_DEF_ERROR_CODE(0, 0x3104)
+#define TSDB_CODE_TSMA_RM_SKEY_IN_HASH           TAOS_DEF_ERROR_CODE(0, 0x3105)
+
+//rsma
+#define TSDB_CODE_RSMA_INVALID_ENV               TAOS_DEF_ERROR_CODE(0, 0x3150)
+#define TSDB_CODE_RSMA_INVALID_STAT              TAOS_DEF_ERROR_CODE(0, 0x3151)
+
+
 #ifdef __cplusplus
 }
 #endif

source/client/src/clientImpl.c

@@ -724,7 +724,6 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) {
                           .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE};
 
       SAppInstInfo* pAppInfo = getAppInfo(pRequest);
-
       if (TSDB_CODE_SUCCESS == code) {
         code = qCreateQueryPlan(&cxt, &pRequest->body.pDag, pNodeList);
         if (code) {
@@ -747,6 +746,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) {
         };      
         code = schedulerAsyncExecJob(&req, &pRequest->body.queryJob);
       } else {
+        tscError("0x%"PRIx64" failed to create query plan, code:%s 0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId);
         pRequest->body.queryFp(pRequest->body.param, pRequest, code);
       }
 

source/client/test/clientTests.cpp

@@ -778,9 +778,9 @@ TEST(testCase, async_api_test) {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   ASSERT_NE(pConn, nullptr);
 
-  taos_query(pConn, "use test");
+  taos_query(pConn, "use nest");
 
-  TAOS_RES* pRes = taos_query(pConn, "desc abc1.tu");
+  TAOS_RES* pRes = taos_query(pConn, "select NOW() from (select * from regular_table_2 where tbname in ('regular_table_2_1') and  q_bigint <= 9223372036854775807 and  q_tinyint <= 127 and    q_bool in ( true , false) ) order by ts;");
   if (taos_errno(pRes) != 0) {
     printf("failed, reason:%s\n", taos_errstr(pRes));
   }

source/common/src/tdatablock.c

@@ -294,7 +294,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, in
 
 int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows) {
   ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type);
-  if (numOfRows == 0) {
+  if (numOfRows <= 0) {
     return numOfRows;
   }
 
@@ -1239,6 +1239,9 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
         return NULL;
       }
 
+      if (pSrc->pData == NULL) {
+        continue;
+      }
       colDataAssign(pDst, pSrc, pDataBlock->info.rows);
     }
 
@@ -1631,25 +1634,31 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
             break;
           default:
             if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) {
-              char tv[8] = {0};
-              if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) {
-                float v = 0;
-                GET_TYPED_DATA(v, float, pColInfoData->info.type, var);
-                SET_TYPED_DATA(&tv, pCol->type, v);
-              } else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) {
-                double v = 0;
-                GET_TYPED_DATA(v, double, pColInfoData->info.type, var);
-                SET_TYPED_DATA(&tv, pCol->type, v);
-              } else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) {
-                int64_t v = 0;
-                GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var);
-                SET_TYPED_DATA(&tv, pCol->type, v);
+              if (pCol->type == pColInfoData->info.type) {
+                tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, var, true, offset,
+                                    k);
               } else {
-                uint64_t v = 0;
-                GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var);
-                SET_TYPED_DATA(&tv, pCol->type, v);
+                char tv[8] = {0};
+                if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) {
+                  float v = 0;
+                  GET_TYPED_DATA(v, float, pColInfoData->info.type, var);
+                  SET_TYPED_DATA(&tv, pCol->type, v);
+                } else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) {
+                  double v = 0;
+                  GET_TYPED_DATA(v, double, pColInfoData->info.type, var);
+                  SET_TYPED_DATA(&tv, pCol->type, v);
+                } else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) {
+                  int64_t v = 0;
+                  GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var);
+                  SET_TYPED_DATA(&tv, pCol->type, v);
+                } else {
+                  uint64_t v = 0;
+                  GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var);
+                  SET_TYPED_DATA(&tv, pCol->type, v);
+                }
+                tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, tv, true, offset,
+                                    k);
               }
-              tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, tv, true, offset, k);
             } else {
               uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type);
               TASSERT(0);

source/common/src/tmsg.c

@@ -3877,9 +3877,10 @@ int32_t tEncodeTSma(SEncoder *pCoder, const STSma *pSma) {
   if (tEncodeCStr(pCoder, pSma->indexName) < 0) return -1;
   if (tEncodeI32(pCoder, pSma->exprLen) < 0) return -1;
   if (tEncodeI32(pCoder, pSma->tagsFilterLen) < 0) return -1;
-  if (tEncodeI32(pCoder, pSma->numOfVgroups) < 0) return -1;
   if (tEncodeI64(pCoder, pSma->indexUid) < 0) return -1;
   if (tEncodeI64(pCoder, pSma->tableUid) < 0) return -1;
+  if (tEncodeI64(pCoder, pSma->dstTbUid) < 0) return -1;
+  if (tEncodeCStr(pCoder, pSma->dstTbName) < 0) return -1;
   if (tEncodeI64(pCoder, pSma->interval) < 0) return -1;
   if (tEncodeI64(pCoder, pSma->offset) < 0) return -1;
   if (tEncodeI64(pCoder, pSma->sliding) < 0) return -1;
@@ -3889,17 +3890,10 @@ int32_t tEncodeTSma(SEncoder *pCoder, const STSma *pSma) {
   if (pSma->tagsFilterLen > 0) {
     if (tEncodeCStr(pCoder, pSma->tagsFilter) < 0) return -1;
   }
-  for (int32_t v = 0; v < pSma->numOfVgroups; ++v) {
-    if (tEncodeI32(pCoder, pSma->pVgEpSet[v].vgId) < 0) return -1;
-    if (tEncodeI8(pCoder, pSma->pVgEpSet[v].epSet.inUse) < 0) return -1;
-    int8_t numOfEps = pSma->pVgEpSet[v].epSet.numOfEps;
-    if (tEncodeI8(pCoder, numOfEps) < 0) return -1;
-    for (int32_t n = 0; n < numOfEps; ++n) {
-      const SEp *pEp = &pSma->pVgEpSet[v].epSet.eps[n];
-      if (tEncodeCStr(pCoder, pEp->fqdn) < 0) return -1;
-      if (tEncodeU16(pCoder, pEp->port) < 0) return -1;
-    }
-  }
+
+  tEncodeSSchemaWrapper(pCoder, &pSma->schemaRow);
+  tEncodeSSchemaWrapper(pCoder, &pSma->schemaTag);
+
   return 0;
 }
 
@@ -3907,14 +3901,15 @@ int32_t tDecodeTSma(SDecoder *pCoder, STSma *pSma) {
   if (tDecodeI8(pCoder, &pSma->version) < 0) return -1;
   if (tDecodeI8(pCoder, &pSma->intervalUnit) < 0) return -1;
   if (tDecodeI8(pCoder, &pSma->slidingUnit) < 0) return -1;
-  if (tDecodeI32(pCoder, &pSma->dstVgId) < 0) return -1;
   if (tDecodeI8(pCoder, &pSma->timezoneInt) < 0) return -1;
+  if (tDecodeI32(pCoder, &pSma->dstVgId) < 0) return -1;
   if (tDecodeCStrTo(pCoder, pSma->indexName) < 0) return -1;
   if (tDecodeI32(pCoder, &pSma->exprLen) < 0) return -1;
   if (tDecodeI32(pCoder, &pSma->tagsFilterLen) < 0) return -1;
-  if (tDecodeI32(pCoder, &pSma->numOfVgroups) < 0) return -1;
   if (tDecodeI64(pCoder, &pSma->indexUid) < 0) return -1;
   if (tDecodeI64(pCoder, &pSma->tableUid) < 0) return -1;
+  if (tDecodeI64(pCoder, &pSma->dstTbUid) < 0) return -1;
+  if (tDecodeCStr(pCoder, &pSma->dstTbName) < 0) return -1;
   if (tDecodeI64(pCoder, &pSma->interval) < 0) return -1;
   if (tDecodeI64(pCoder, &pSma->offset) < 0) return -1;
   if (tDecodeI64(pCoder, &pSma->sliding) < 0) return -1;
@@ -3928,27 +3923,9 @@ int32_t tDecodeTSma(SDecoder *pCoder, STSma *pSma) {
   } else {
     pSma->tagsFilter = NULL;
   }
-  if (pSma->numOfVgroups > 0) {
-    pSma->pVgEpSet = (SVgEpSet *)tDecoderMalloc(pCoder, pSma->numOfVgroups * sizeof(SVgEpSet));
-    if (!pSma->pVgEpSet) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      return -1;
-    }
-
-    memset(pSma->pVgEpSet, 0, pSma->numOfVgroups * sizeof(SVgEpSet));
-
-    for (int32_t v = 0; v < pSma->numOfVgroups; ++v) {
-      if (tDecodeI32(pCoder, &pSma->pVgEpSet[v].vgId) < 0) return -1;
-      if (tDecodeI8(pCoder, &pSma->pVgEpSet[v].epSet.inUse) < 0) return -1;
-      if (tDecodeI8(pCoder, &pSma->pVgEpSet[v].epSet.numOfEps) < 0) return -1;
-      int8_t numOfEps = pSma->pVgEpSet[v].epSet.numOfEps;
-      for (int32_t n = 0; n < numOfEps; ++n) {
-        SEp *pEp = &pSma->pVgEpSet[v].epSet.eps[n];
-        if (tDecodeCStrTo(pCoder, pEp->fqdn) < 0) return -1;
-        if (tDecodeU16(pCoder, &pEp->port) < 0) return -1;
-      }
-    }
-  }
+  // only needed in dstVgroup
+  tDecodeSSchemaWrapperEx(pCoder, &pSma->schemaRow);
+  tDecodeSSchemaWrapperEx(pCoder, &pSma->schemaTag);
 
   return 0;
 }
@@ -3991,55 +3968,6 @@ int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) {
   return 0;
 }
 
-int32_t tEncodeSVGetTSmaExpWndsReq(SEncoder *pCoder, const SVGetTsmaExpWndsReq *pReq) {
-  if (tStartEncode(pCoder) < 0) return -1;
-
-  if (tEncodeI64(pCoder, pReq->indexUid) < 0) return -1;
-  if (tEncodeI64(pCoder, pReq->queryWindow.skey) < 0) return -1;
-  if (tEncodeI64(pCoder, pReq->queryWindow.ekey) < 0) return -1;
-
-  tEndEncode(pCoder);
-  return 0;
-}
-
-int32_t tDecodeSVGetTsmaExpWndsReq(SDecoder *pCoder, SVGetTsmaExpWndsReq *pReq) {
-  if (tStartDecode(pCoder) < 0) return -1;
-
-  if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1;
-  if (tDecodeI64(pCoder, &pReq->queryWindow.skey) < 0) return -1;
-  if (tDecodeI64(pCoder, &pReq->queryWindow.ekey) < 0) return -1;
-
-  tEndDecode(pCoder);
-  return 0;
-}
-
-int32_t tEncodeSVGetTSmaExpWndsRsp(SEncoder *pCoder, const SVGetTsmaExpWndsRsp *pReq) {
-  if (tStartEncode(pCoder) < 0) return -1;
-
-  if (tEncodeI64(pCoder, pReq->indexUid) < 0) return -1;
-  if (tEncodeI8(pCoder, pReq->flags) < 0) return -1;
-  if (tEncodeI32(pCoder, pReq->numExpWnds) < 0) return -1;
-  for (int32_t i = 0; i < pReq->numExpWnds; ++i) {
-    if (tEncodeI64(pCoder, pReq->wndSKeys[i]) < 0) return -1;
-  }
-  tEndEncode(pCoder);
-  return 0;
-}
-
-int32_t tDecodeSVGetTsmaExpWndsRsp(SDecoder *pCoder, SVGetTsmaExpWndsRsp *pReq) {
-  if (tStartDecode(pCoder) < 0) return -1;
-
-  if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1;
-  if (tDecodeI8(pCoder, &pReq->flags) < 0) return -1;
-  if (tDecodeI32(pCoder, &pReq->numExpWnds) < 0) return -1;
-  for (int32_t i = 0; i < pReq->numExpWnds; ++i) {
-    if (tDecodeI64(pCoder, &pReq->wndSKeys[i]) < 0) return -1;
-  }
-
-  tEndDecode(pCoder);
-  return 0;
-}
-
 int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) {
   int32_t headLen = sizeof(SMsgHead);
   if (buf != NULL) {

source/dnode/mgmt/mgmt_dnode/src/dmHandle.c

@@ -38,9 +38,13 @@ static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) {
     SStatusRsp statusRsp = {0};
     if (pRsp->pCont != NULL && pRsp->contLen > 0 &&
         tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) {
-      pMgmt->pData->dnodeVer = statusRsp.dnodeVer;
-      dmUpdateDnodeCfg(pMgmt, &statusRsp.dnodeCfg);
-      dmUpdateEps(pMgmt->pData, statusRsp.pDnodeEps);
+      dTrace("status msg received from mnode, dnodeVer:%" PRId64 " saved:%" PRId64, statusRsp.dnodeVer,
+             pMgmt->pData->dnodeVer);
+      if (pMgmt->pData->dnodeVer != statusRsp.dnodeVer) {
+        pMgmt->pData->dnodeVer = statusRsp.dnodeVer;
+        dmUpdateDnodeCfg(pMgmt, &statusRsp.dnodeCfg);
+        dmUpdateEps(pMgmt->pData, statusRsp.pDnodeEps);
+      }
     }
     rpcFreeCont(pRsp->pCont);
     tFreeSStatusRsp(&statusRsp);
@@ -89,7 +93,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
   SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .info.ahandle = (void *)0x9527};
   SRpcMsg rpcRsp = {0};
 
-  dTrace("send status msg to mnode");
+  dTrace("send status msg to mnode, dnodeVer:%" PRId64, req.dnodeVer);
 
   SEpSet epSet = {0};
   dmGetMnodeEpSet(pMgmt->pData, &epSet);

source/dnode/mgmt/node_util/src/dmEps.c

@@ -81,6 +81,13 @@ int32_t dmReadEps(SDnodeData *pData) {
   }
   pData->dnodeId = dnodeId->valueint;
 
+  cJSON *dnodeVer = cJSON_GetObjectItem(root, "dnodeVer");
+  if (!dnodeVer || dnodeVer->type != cJSON_String) {
+    dError("failed to read %s since dnodeVer not found", file);
+    goto _OVER;
+  }
+  pData->dnodeVer = atoll(dnodeVer->valuestring);
+
   cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId");
   if (!clusterId || clusterId->type != cJSON_String) {
     dError("failed to read %s since clusterId not found", file);
@@ -193,6 +200,7 @@ int32_t dmWriteEps(SDnodeData *pData) {
 
   len += snprintf(content + len, maxLen - len, "{\n");
   len += snprintf(content + len, maxLen - len, "  \"dnodeId\": %d,\n", pData->dnodeId);
+  len += snprintf(content + len, maxLen - len, "  \"dnodeVer\": \"%" PRId64 "\",\n", pData->dnodeVer);
   len += snprintf(content + len, maxLen - len, "  \"clusterId\": \"%" PRId64 "\",\n", pData->clusterId);
   len += snprintf(content + len, maxLen - len, "  \"dropped\": %d,\n", pData->dropped);
   len += snprintf(content + len, maxLen - len, "  \"dnodes\": [{\n");
@@ -224,30 +232,15 @@ int32_t dmWriteEps(SDnodeData *pData) {
   }
 
   pData->updateTime = taosGetTimestampMs();
-  dDebug("successed to write %s", realfile);
+  dDebug("successed to write %s, dnodeVer:%" PRId64, realfile, pData->dnodeVer);
   return 0;
 }
 
 void dmUpdateEps(SDnodeData *pData, SArray *eps) {
-  int32_t numOfEps = taosArrayGetSize(eps);
-  if (numOfEps <= 0) return;
-
   taosThreadRwlockWrlock(&pData->lock);
-
-  int32_t numOfEpsOld = (int32_t)taosArrayGetSize(pData->dnodeEps);
-  if (numOfEps != numOfEpsOld) {
-    dDebug("new dnode list get from mnode");
-    dmResetEps(pData, eps);
-    dmWriteEps(pData);
-  } else {
-    int32_t size = numOfEps * sizeof(SDnodeEp);
-    if (memcmp(pData->dnodeEps->pData, eps->pData, size) != 0) {
-      dDebug("new dnode list get from mnode");
-      dmResetEps(pData, eps);
-      dmWriteEps(pData);
-    }
-  }
-
+  dDebug("new dnode list get from mnode, dnodeVer:%" PRId64, pData->dnodeVer);
+  dmResetEps(pData, eps);
+  dmWriteEps(pData);
   taosThreadRwlockUnlock(&pData->lock);
 }
 

source/dnode/mnode/impl/inc/mndDef.h

@@ -298,31 +298,32 @@ typedef struct {
 } SVgObj;
 
 typedef struct {
-  char      name[TSDB_TABLE_FNAME_LEN];
-  char      stb[TSDB_TABLE_FNAME_LEN];
-  char      db[TSDB_DB_FNAME_LEN];
-  int64_t   createdTime;
-  int64_t   uid;
-  int64_t   stbUid;
-  int64_t   dbUid;
-  int8_t    intervalUnit;
-  int8_t    slidingUnit;
-  int8_t    timezone;
-  int32_t   dstVgId;  // for stream
-  int64_t   dstTbUid;
-  int64_t   interval;
-  int64_t   offset;
-  int64_t   sliding;
-  int32_t   exprLen;  // strlen + 1
-  int32_t   tagsFilterLen;
-  int32_t   sqlLen;
-  int32_t   astLen;
-  int32_t   numOfVgroups;
-  char*     expr;
-  char*     tagsFilter;
-  char*     sql;
-  char*     ast;
-  SVgEpSet* pVgEpSet;
+  char           name[TSDB_TABLE_FNAME_LEN];
+  char           stb[TSDB_TABLE_FNAME_LEN];
+  char           db[TSDB_DB_FNAME_LEN];
+  char           dstTbName[TSDB_TABLE_FNAME_LEN];
+  int64_t        createdTime;
+  int64_t        uid;
+  int64_t        stbUid;
+  int64_t        dbUid;
+  int64_t        dstTbUid;
+  int8_t         intervalUnit;
+  int8_t         slidingUnit;
+  int8_t         timezone;
+  int32_t        dstVgId;  // for stream
+  int64_t        interval;
+  int64_t        offset;
+  int64_t        sliding;
+  int32_t        exprLen;  // strlen + 1
+  int32_t        tagsFilterLen;
+  int32_t        sqlLen;
+  int32_t        astLen;
+  char*          expr;
+  char*          tagsFilter;
+  char*          sql;
+  char*          ast;
+  SSchemaWrapper schemaRow;  // for dstVgroup
+  SSchemaWrapper schemaTag;  // for dstVgroup
 } SSmaObj;
 
 typedef struct {

source/dnode/mnode/impl/inc/mndScheduler.h

@@ -27,10 +27,10 @@ void    mndCleanupScheduler(SMnode* pMnode);
 
 int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub);
 
-int32_t mndScheduleStream1(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream);
+int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream);
 
-int32_t mndConvertRSmaTask(const char* ast, int64_t uid, int8_t triggerType, int64_t watermark, char** pStr,
-                           int32_t* pLen, double filesFactor);
+int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType,
+                           int64_t watermark, double filesFactor);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/inc/mndTrans.h

@@ -75,6 +75,7 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans);
 int32_t mndTransProcessRsp(SRpcMsg *pRsp);
 void    mndTransPullup(SMnode *pMnode);
 int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans);
+void    mndTransExecute(SMnode *pMnode, STrans *pTrans);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/src/mndConsumer.c

@@ -447,6 +447,8 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
 
   pConsumerOld = mndAcquireConsumer(pMnode, consumerId);
   if (pConsumerOld == NULL) {
+    mInfo("receive subscribe request from new consumer: %ld", consumerId);
+
     pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup);
     tstrncpy(pConsumerNew->clientId, subscribe.clientId, 256);
     pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY;
@@ -463,7 +465,12 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
 
   } else {
     /*taosRLockLatch(&pConsumerOld->lock);*/
+
     int32_t status = atomic_load_32(&pConsumerOld->status);
+
+    mInfo("receive subscribe request from old consumer: %ld, current status: %s", consumerId,
+          mndConsumerStatusName(status));
+
     if (status != MQ_CONSUMER_STATUS__READY) {
       terrno = TSDB_CODE_MND_CONSUMER_NOT_READY;
       goto SUBSCRIBE_OVER;

source/dnode/mnode/impl/src/mndDnode.c

@@ -385,7 +385,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
   int64_t dnodeVer = sdbGetTableVer(pMnode->pSdb, SDB_DNODE) + sdbGetTableVer(pMnode->pSdb, SDB_MNODE);
   int64_t curMs = taosGetTimestampMs();
   bool    online = mndIsDnodeOnline(pDnode, curMs);
-  bool    dnodeChanged = (statusReq.dnodeVer != dnodeVer);
+  bool    dnodeChanged = (statusReq.dnodeVer == 0) || (statusReq.dnodeVer != dnodeVer);
   bool    reboot = (pDnode->rebootTime != statusReq.rebootTime);
   bool    needCheck = !online || dnodeChanged || reboot;
 
@@ -427,7 +427,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
     if (!online) {
       mInfo("dnode:%d, from offline to online", pDnode->id);
     } else {
-      mDebug("dnode:%d, send dnode epset, online:%d dnode_ver:%" PRId64 ":%" PRId64 " reboot:%d", pDnode->id, online,
+      mDebug("dnode:%d, send dnode epset, online:%d dnodeVer:%" PRId64 ":%" PRId64 " reboot:%d", pDnode->id, online,
              statusReq.dnodeVer, dnodeVer, reboot);
     }
 

source/dnode/mnode/impl/src/mndOffset.c

@@ -183,7 +183,10 @@ static int32_t mndProcessCommitOffsetReq(SRpcMsg *pMsg) {
 
   for (int32_t i = 0; i < commitOffsetReq.num; i++) {
     SMqOffset *pOffset = &commitOffsetReq.offsets[i];
+    mInfo("commit offset %ld to vg %d of consumer group %s on topic %s", pOffset->offset, pOffset->vgId,
+          pOffset->cgroup, pOffset->topicName);
     if (mndMakePartitionKey(key, pOffset->cgroup, pOffset->topicName, pOffset->vgId) < 0) {
+      mError("submit offset to topic %s failed", pOffset->topicName);
       return -1;
     }
     bool          create = false;
@@ -192,7 +195,7 @@ static int32_t mndProcessCommitOffsetReq(SRpcMsg *pMsg) {
       SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pOffset->topicName);
       if (pTopic == NULL) {
         terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST;
-        mError("submit offset to topic %s failed since  %s", pOffset->topicName, terrstr());
+        mError("submit offset to topic %s failed since %s", pOffset->topicName, terrstr());
         continue;
       }
       pOffsetObj = taosMemoryMalloc(sizeof(SMqOffsetObj));

source/dnode/mnode/impl/src/mndScheduler.c

@@ -42,8 +42,8 @@ static int32_t mndAddTaskToTaskSet(SArray* pArray, SStreamTask* pTask) {
   return 0;
 }
 
-int32_t mndConvertRSmaTask(const char* ast, int64_t uid, int8_t triggerType, int64_t watermark, char** pStr,
-                           int32_t* pLen, double filesFactor) {
+int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType,
+                           int64_t watermark, double filesFactor) {
   SNode*      pAst = NULL;
   SQueryPlan* pPlan = NULL;
   terrno = TSDB_CODE_SUCCESS;
@@ -53,7 +53,7 @@ int32_t mndConvertRSmaTask(const char* ast, int64_t uid, int8_t triggerType, int
     goto END;
   }
 
-  if (qSetSTableIdForRSma(pAst, uid) < 0) {
+  if (qSetSTableIdForRsma(pAst, uid) < 0) {
     terrno = TSDB_CODE_QRY_INVALID_INPUT;
     goto END;
   }
@@ -86,7 +86,7 @@ int32_t mndConvertRSmaTask(const char* ast, int64_t uid, int8_t triggerType, int
   }
 
   SSubplan* plan = nodesListGetNode(inner->pNodeList, 0);
-  if (qSubPlanToString(plan, pStr, pLen) < 0) {
+  if (qSubPlanToString(plan, pDst, pDstLen) < 0) {
     terrno = TSDB_CODE_QRY_INVALID_INPUT;
     goto END;
   }
@@ -151,33 +151,36 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, STrans* pTrans, SStreamObj*
     ASSERT(pDb);
 
     if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) {
-      sdbRelease(pMnode->pSdb, pDb);
-
-      SArray* pVgs = pTask->shuffleDispatcher.dbInfo.pVgroupInfos;
-      int32_t sz = taosArrayGetSize(pVgs);
-      SArray* sinkLv = taosArrayGetP(pStream->tasks, 0);
-      int32_t sinkLvSize = taosArrayGetSize(sinkLv);
-      for (int32_t i = 0; i < sz; i++) {
-        SVgroupInfo* pVgInfo = taosArrayGet(pVgs, i);
-        for (int32_t j = 0; j < sinkLvSize; j++) {
-          SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j);
-          if (pLastLevelTask->nodeId == pVgInfo->vgId) {
-            pVgInfo->taskId = pLastLevelTask->taskId;
-            break;
-          }
+      ASSERT(0);
+      return -1;
+    }
+    sdbRelease(pMnode->pSdb, pDb);
+
+    SArray* pVgs = pTask->shuffleDispatcher.dbInfo.pVgroupInfos;
+    int32_t sz = taosArrayGetSize(pVgs);
+    SArray* sinkLv = taosArrayGetP(pStream->tasks, 0);
+    int32_t sinkLvSize = taosArrayGetSize(sinkLv);
+    for (int32_t i = 0; i < sz; i++) {
+      SVgroupInfo* pVgInfo = taosArrayGet(pVgs, i);
+      for (int32_t j = 0; j < sinkLvSize; j++) {
+        SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j);
+        if (pLastLevelTask->nodeId == pVgInfo->vgId) {
+          pVgInfo->taskId = pLastLevelTask->taskId;
+          ASSERT(pVgInfo->taskId != 0);
+          break;
         }
       }
-    } else {
-      pTask->dispatchType = TASK_DISPATCH__FIXED;
-      pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH;
-      SArray* pArray = taosArrayGetP(pStream->tasks, 0);
-      // one sink only
-      ASSERT(taosArrayGetSize(pArray) == 1);
-      SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0);
-      pTask->fixedEpDispatcher.taskId = lastLevelTask->taskId;
-      pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId;
-      pTask->fixedEpDispatcher.epSet = lastLevelTask->epSet;
     }
+  } else {
+    pTask->dispatchType = TASK_DISPATCH__FIXED;
+    pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH;
+    SArray* pArray = taosArrayGetP(pStream->tasks, 0);
+    // one sink only
+    ASSERT(taosArrayGetSize(pArray) == 1);
+    SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0);
+    pTask->fixedEpDispatcher.taskId = lastLevelTask->taskId;
+    pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId;
+    pTask->fixedEpDispatcher.epSet = lastLevelTask->epSet;
   }
   return 0;
 }
@@ -335,7 +338,7 @@ int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, STrans* pTrans, SStreamObj*
   return 0;
 }
 
-int32_t mndScheduleStream1(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
+int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
   SSdb*       pSdb = pMnode->pSdb;
   SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan);
   if (pPlan == NULL) {
@@ -361,6 +364,7 @@ int32_t mndScheduleStream1(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream)
       mndAddFixedSinkTaskToStream(pMnode, pTrans, pStream);
     }
   }
+
   if (totLevel > 1) {
     SStreamTask* pFinalTask;
     // inner plan
@@ -378,7 +382,10 @@ int32_t mndScheduleStream1(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream)
       pFinalTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
 
       // dispatch
-      mndAddDispatcherToInnerTask(pMnode, pTrans, pStream, pFinalTask);
+      if (mndAddDispatcherToInnerTask(pMnode, pTrans, pStream, pFinalTask) < 0) {
+        qDestroyQueryPlan(pPlan);
+        return -1;
+      }
 
       // exec
       pFinalTask->execType = TASK_EXEC__PIPE;
@@ -472,229 +479,7 @@ int32_t mndScheduleStream1(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream)
       }
     }
   }
-  return 0;
-}
-
-int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
-  SSdb*       pSdb = pMnode->pSdb;
-  SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan);
-  if (pPlan == NULL) {
-    terrno = TSDB_CODE_QRY_INVALID_INPUT;
-    return -1;
-  }
-  ASSERT(pStream->vgNum == 0);
-
-  int32_t totLevel = LIST_LENGTH(pPlan->pSubplans);
-  ASSERT(totLevel <= 2);
-  pStream->tasks = taosArrayInit(totLevel, sizeof(void*));
-
-  bool hasExtraSink = false;
-  bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0;
-  if (totLevel == 2 || externalTargetDB) {
-    SArray* taskOneLevel = taosArrayInit(0, sizeof(void*));
-    taosArrayPush(pStream->tasks, &taskOneLevel);
-    // add extra sink
-    hasExtraSink = true;
-    if (pStream->fixedSinkVgId == 0) {
-      mndAddShuffleSinkTasksToStream(pMnode, pTrans, pStream);
-    } else {
-      mndAddFixedSinkTaskToStream(pMnode, pTrans, pStream);
-    }
-  }
-
-  for (int32_t level = 0; level < totLevel; level++) {
-    SArray* taskOneLevel = taosArrayInit(0, sizeof(void*));
-    taosArrayPush(pStream->tasks, &taskOneLevel);
-    SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, level);
-    ASSERT(LIST_LENGTH(inner->pNodeList) == 1);
-
-    SSubplan* plan = nodesListGetNode(inner->pNodeList, 0);
-
-    // if (level == totLevel - 1 /* or no snode */) {
-    if (level == totLevel - 1) {
-      // last level, source, must assign to vnode
-      // must be scan type
-      ASSERT(plan->subplanType == SUBPLAN_TYPE_SCAN);
-
-      // replicate task to each vnode
-      void* pIter = NULL;
-      while (1) {
-        SVgObj* pVgroup;
-        pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup);
-        if (pIter == NULL) break;
-        if (pVgroup->dbUid != pStream->dbUid) {
-          sdbRelease(pSdb, pVgroup);
-          continue;
-        }
-        SStreamTask* pTask = tNewSStreamTask(pStream->uid);
-        mndAddTaskToTaskSet(taskOneLevel, pTask);
-        // source part
-        pTask->sourceType = TASK_SOURCE__SCAN;
-        pTask->inputType = TASK_INPUT_TYPE__SUMBIT_BLOCK;
-
-        // sink part
-        if (level == 0) {
-          // only for inplace
-          pTask->sinkType = TASK_SINK__NONE;
-          if (!hasExtraSink) {
-#if 1
-            if (pStream->createdBy == STREAM_CREATED_BY__SMA) {
-              pTask->sinkType = TASK_SINK__SMA;
-              pTask->smaSink.smaId = pStream->smaId;
-            } else {
-              pTask->sinkType = TASK_SINK__TABLE;
-              pTask->tbSink.stbUid = pStream->targetStbUid;
-              memcpy(pTask->tbSink.stbFullName, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN);
-              pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema);
-            }
-#endif
-          }
-        } else {
-          pTask->sinkType = TASK_SINK__NONE;
-        }
-
-        // dispatch part
-        if (level == 0 && !hasExtraSink) {
-          pTask->dispatchType = TASK_DISPATCH__NONE;
-        } else {
-          // add fixed ep dispatcher
-          int32_t lastLevel = level - 1;
-          if (hasExtraSink) lastLevel++;
-          ASSERT(lastLevel == 0);
-          SArray* pArray = taosArrayGetP(pStream->tasks, lastLevel);
-          // one merge only
-          ASSERT(taosArrayGetSize(pArray) == 1);
-          SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0);
-          pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH;
-          pTask->dispatchType = TASK_DISPATCH__FIXED;
-
-          pTask->fixedEpDispatcher.taskId = lastLevelTask->taskId;
-          pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId;
-          pTask->fixedEpDispatcher.epSet = lastLevelTask->epSet;
-        }
-
-        // exec part
-        pTask->execType = TASK_EXEC__PIPE;
-        if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) {
-          sdbRelease(pSdb, pVgroup);
-          qDestroyQueryPlan(pPlan);
-          return -1;
-        }
-        sdbRelease(pSdb, pVgroup);
-      }
-    } else {
-      // merge plan
-
-      // TODO if has snode, assign to snode
-
-      // else, assign to vnode
-      ASSERT(plan->subplanType == SUBPLAN_TYPE_MERGE);
-      SStreamTask* pTask = tNewSStreamTask(pStream->uid);
-      mndAddTaskToTaskSet(taskOneLevel, pTask);
-
-      // source part, currently only support multi source
-      pTask->sourceType = TASK_SOURCE__PIPE;
-      pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
-
-      // sink part
-      pTask->sinkType = TASK_SINK__NONE;
-
-      // dispatch part
-      ASSERT(hasExtraSink);
-      /*pTask->dispatchType = TASK_DISPATCH__NONE;*/
-#if 1
-
-      if (hasExtraSink) {
-        // add dispatcher
-        if (pStream->fixedSinkVgId == 0) {
-          pTask->dispatchType = TASK_DISPATCH__SHUFFLE;
-
-          pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH;
-          SDbObj* pDb = mndAcquireDb(pMnode, pStream->targetDb);
-          ASSERT(pDb);
-          if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) {
-            sdbRelease(pSdb, pDb);
-            qDestroyQueryPlan(pPlan);
-            return -1;
-          }
-          sdbRelease(pSdb, pDb);
-
-          // put taskId to useDbRsp
-          // TODO: optimize
-          SArray* pVgs = pTask->shuffleDispatcher.dbInfo.pVgroupInfos;
-          int32_t sz = taosArrayGetSize(pVgs);
-          SArray* sinkLv = taosArrayGetP(pStream->tasks, 0);
-          int32_t sinkLvSize = taosArrayGetSize(sinkLv);
-          for (int32_t i = 0; i < sz; i++) {
-            SVgroupInfo* pVgInfo = taosArrayGet(pVgs, i);
-            for (int32_t j = 0; j < sinkLvSize; j++) {
-              SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j);
-              if (pLastLevelTask->nodeId == pVgInfo->vgId) {
-                pVgInfo->taskId = pLastLevelTask->taskId;
-                break;
-              }
-            }
-          }
-        } else {
-          pTask->dispatchType = TASK_DISPATCH__FIXED;
-          /*pTask->dispatchMsgType = TDMT_VND_TASK_WRITE_EXEC;*/
-          pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH;
-          SArray* pArray = taosArrayGetP(pStream->tasks, 0);
-          // one sink only
-          ASSERT(taosArrayGetSize(pArray) == 1);
-          SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0);
-          pTask->fixedEpDispatcher.taskId = lastLevelTask->taskId;
-          pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId;
-          pTask->fixedEpDispatcher.epSet = lastLevelTask->epSet;
-        }
-      }
-#endif
-
-      // exec part
-      pTask->execType = TASK_EXEC__MERGE;
-      SVgObj* pVgroup = mndSchedFetchOneVg(pMnode, pStream->dbUid);
-      ASSERT(pVgroup);
-      if (mndAssignTaskToVg(pMnode, pTrans, pTask, plan, pVgroup) < 0) {
-        sdbRelease(pSdb, pVgroup);
-        qDestroyQueryPlan(pPlan);
-        return -1;
-      }
-      sdbRelease(pSdb, pVgroup);
-    }
-  }
-
-#if 0
-  if (totLevel == 2) {
-    void* pIter = NULL;
-    while (1) {
-      SVgObj* pVgroup;
-      pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup);
-      if (pIter == NULL) break;
-      if (pVgroup->dbUid != pStream->dbUid) {
-        sdbRelease(pSdb, pVgroup);
-        continue;
-      }
-      SStreamTask* pTask = tNewSStreamTask(pStream->uid);
-
-      // source part
-      pTask->sourceType = TASK_SOURCE__MERGE;
-      pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK;
-
-      // sink part
-      pTask->sinkType = TASK_SINK__NONE;
-
-      // dispatch part
-      pTask->dispatchType = TASK_DISPATCH__NONE;
-
-      // exec part
-      pTask->execType = TASK_EXEC__NONE;
-    }
-  }
-#endif
-
-  // free memory
   qDestroyQueryPlan(pPlan);
-
   return 0;
 }
 

source/dnode/mnode/impl/src/mndSma.c

@@ -26,6 +26,7 @@
 #include "mndTrans.h"
 #include "mndUser.h"
 #include "mndVgroup.h"
+#include "parser.h"
 #include "tname.h"
 
 #define TSDB_SMA_VER_NUMBER   1
@@ -82,10 +83,12 @@ static SSdbRaw *mndSmaActionEncode(SSmaObj *pSma) {
   SDB_SET_BINARY(pRaw, dataPos, pSma->name, TSDB_TABLE_FNAME_LEN, _OVER)
   SDB_SET_BINARY(pRaw, dataPos, pSma->stb, TSDB_TABLE_FNAME_LEN, _OVER)
   SDB_SET_BINARY(pRaw, dataPos, pSma->db, TSDB_DB_FNAME_LEN, _OVER)
+  SDB_SET_BINARY(pRaw, dataPos, pSma->dstTbName, TSDB_DB_FNAME_LEN, _OVER)
   SDB_SET_INT64(pRaw, dataPos, pSma->createdTime, _OVER)
   SDB_SET_INT64(pRaw, dataPos, pSma->uid, _OVER)
   SDB_SET_INT64(pRaw, dataPos, pSma->stbUid, _OVER)
   SDB_SET_INT64(pRaw, dataPos, pSma->dbUid, _OVER)
+  SDB_SET_INT64(pRaw, dataPos, pSma->dstTbUid, _OVER)
   SDB_SET_INT8(pRaw, dataPos, pSma->intervalUnit, _OVER)
   SDB_SET_INT8(pRaw, dataPos, pSma->slidingUnit, _OVER)
   SDB_SET_INT8(pRaw, dataPos, pSma->timezone, _OVER)
@@ -147,10 +150,12 @@ static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) {
   SDB_GET_BINARY(pRaw, dataPos, pSma->name, TSDB_TABLE_FNAME_LEN, _OVER)
   SDB_GET_BINARY(pRaw, dataPos, pSma->stb, TSDB_TABLE_FNAME_LEN, _OVER)
   SDB_GET_BINARY(pRaw, dataPos, pSma->db, TSDB_DB_FNAME_LEN, _OVER)
+  SDB_GET_BINARY(pRaw, dataPos, pSma->dstTbName, TSDB_DB_FNAME_LEN, _OVER)
   SDB_GET_INT64(pRaw, dataPos, &pSma->createdTime, _OVER)
   SDB_GET_INT64(pRaw, dataPos, &pSma->uid, _OVER)
   SDB_GET_INT64(pRaw, dataPos, &pSma->stbUid, _OVER)
   SDB_GET_INT64(pRaw, dataPos, &pSma->dbUid, _OVER)
+  SDB_GET_INT64(pRaw, dataPos, &pSma->dstTbUid, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &pSma->intervalUnit, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &pSma->slidingUnit, _OVER)
   SDB_GET_INT8(pRaw, dataPos, &pSma->timezone, _OVER)
@@ -260,14 +265,17 @@ static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSm
   req.tagsFilterLen = pSma->tagsFilterLen;
   req.indexUid = pSma->uid;
   req.tableUid = pSma->stbUid;
+  req.dstVgId = pSma->dstVgId;
+  req.dstTbUid = pSma->dstTbUid;
   req.interval = pSma->interval;
   req.offset = pSma->offset;
   req.sliding = pSma->sliding;
   req.expr = pSma->expr;
   req.tagsFilter = pSma->tagsFilter;
-  req.numOfVgroups = pSma->numOfVgroups;
-  req.pVgEpSet = pSma->pVgEpSet;
-  
+  req.schemaRow = pSma->schemaRow;
+  req.schemaTag = pSma->schemaTag;
+  req.dstTbName = pSma->dstTbName;
+
   // get length
   int32_t ret = 0;
   tEncodeSize(tEncodeSVCreateTSmaReq, &req, contLen, ret);
@@ -425,14 +433,30 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans,
   mndReleaseDnode(pMnode, pDnode);
 
   // todo add sma info here
-  SVgEpSet *pVgEpSet = NULL;
-  int32_t   numOfVgroups = 0;
-  if (mndSmaGetVgEpSet(pMnode, pDb, &pVgEpSet, &numOfVgroups) != 0) {
+  SNode *pAst = NULL;
+  if (nodesStringToNode(pSma->ast, &pAst) < 0) {
+    return -1;
+  }
+  if (qExtractResultSchema(pAst, &pSma->schemaRow.nCols, &pSma->schemaRow.pSchema) != 0) {
+    nodesDestroyNode(pAst);
     return -1;
   }
+  nodesDestroyNode(pAst);
+  pSma->schemaRow.version = 1;
+
+  // TODO: the schemaTag generated by qExtractResultXXX later.
+  pSma->schemaTag.nCols = 1;
+  pSma->schemaTag.version = 1;
+  pSma->schemaTag.pSchema = taosMemoryCalloc(1, sizeof(SSchema));
+  if (!pSma->schemaTag.pSchema) {
+    return -1;
+  }
+  pSma->schemaTag.pSchema[0].type = TSDB_DATA_TYPE_BIGINT;
+  pSma->schemaTag.pSchema[0].bytes = TYPE_BYTES[TSDB_DATA_TYPE_BIGINT];
+  pSma->schemaTag.pSchema[0].colId = pSma->schemaRow.nCols + PRIMARYKEY_TIMESTAMP_COL_ID;
+  pSma->schemaTag.pSchema[0].flags = 0;
+  snprintf(pSma->schemaTag.pSchema[0].name, TSDB_COL_NAME_LEN, "groupId");
 
-  pSma->pVgEpSet = pVgEpSet;
-  pSma->numOfVgroups = numOfVgroups;
 
   int32_t smaContLen = 0;
   void   *pSmaReq = mndBuildVCreateSmaReq(pMnode, pVgroup, pSma, &smaContLen);
@@ -464,12 +488,15 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
   memcpy(smaObj.db, pDb->name, TSDB_DB_FNAME_LEN);
   smaObj.createdTime = taosGetTimestampMs();
   smaObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
+  char resultTbName[TSDB_TABLE_FNAME_LEN + 16] = {0};
+  snprintf(resultTbName, TSDB_TABLE_FNAME_LEN + 16, "td.tsma.rst.tb.%s", pCreate->name);
+  memcpy(smaObj.dstTbName, resultTbName, TSDB_TABLE_FNAME_LEN);
+  smaObj.dstTbUid = mndGenerateUid(smaObj.dstTbName, TSDB_TABLE_FNAME_LEN);
   smaObj.stbUid = pStb->uid;
   smaObj.dbUid = pStb->dbUid;
   smaObj.intervalUnit = pCreate->intervalUnit;
   smaObj.slidingUnit = pCreate->slidingUnit;
   smaObj.timezone = pCreate->timezone;
-  smaObj.dstVgId = pCreate->dstVgId;
   smaObj.interval = pCreate->interval;
   smaObj.offset = pCreate->offset;
   smaObj.sliding = pCreate->sliding;
@@ -1087,53 +1114,4 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
 static void mndCancelGetNextSma(SMnode *pMnode, void *pIter) {
   SSdb *pSdb = pMnode->pSdb;
   sdbCancelFetch(pSdb, pIter);
-}
-
+}
\ No newline at end of file
-static int32_t mndSmaGetVgEpSet(SMnode *pMnode, SDbObj *pDb, SVgEpSet **ppVgEpSet, int32_t *numOfVgroups) {
-  SSdb     *pSdb = pMnode->pSdb;
-  SVgObj   *pVgroup = NULL;
-  void     *pIter = NULL;
-  SVgEpSet *pVgEpSet = NULL;
-  int32_t   nAllocVgs = 16;
-  int32_t   nVgs = 0;
-
-  pVgEpSet = taosMemoryCalloc(nAllocVgs, sizeof(SVgEpSet));
-  if (!pVgEpSet) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  while (1) {
-    pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
-    if (pIter == NULL) break;
-    if (pVgroup->dbUid != pDb->uid) {
-      sdbRelease(pSdb, pVgroup);
-      continue;
-    }
-
-    if (nVgs >= nAllocVgs) {
-      void *p = taosMemoryRealloc(pVgEpSet, nAllocVgs * 2 * sizeof(SVgEpSet));
-      if (!p) {
-        taosMemoryFree(pVgEpSet);
-        sdbCancelFetch(pSdb, pIter);
-        sdbRelease(pSdb, pVgroup);
-        terrno = TSDB_CODE_OUT_OF_MEMORY;
-        return -1;
-      }
-      pVgEpSet = (SVgEpSet *)p;
-      nAllocVgs *= 2;
-    }
-
-    (pVgEpSet + nVgs)->vgId = pVgroup->vgId;
-    (pVgEpSet + nVgs)->epSet = mndGetVgroupEpset(pMnode, pVgroup);
-
-    ++nVgs;
-
-    sdbRelease(pSdb, pVgroup);
-  }
-
-  *ppVgEpSet = pVgEpSet;
-  *numOfVgroups = nVgs;
-
-  return 0;
-}

source/dnode/mnode/impl/src/mndStb.c

@@ -395,13 +395,13 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
     req.pRSmaParam.xFilesFactor = pStb->xFilesFactor;
     req.pRSmaParam.delay = pStb->delay;
     if (pStb->ast1Len > 0) {
-      if (mndConvertRSmaTask(pStb->pAst1, pStb->uid, STREAM_TRIGGER_AT_ONCE, 0, &req.pRSmaParam.qmsg1, &req.pRSmaParam.qmsg1Len,
+      if (mndConvertRsmaTask(&req.pRSmaParam.qmsg1, &req.pRSmaParam.qmsg1Len, pStb->pAst1, pStb->uid, STREAM_TRIGGER_AT_ONCE, 0,
                              req.pRSmaParam.xFilesFactor) != TSDB_CODE_SUCCESS) {
         return NULL;
       }
     }
     if (pStb->ast2Len > 0) {
-      if (mndConvertRSmaTask(pStb->pAst2, pStb->uid, STREAM_TRIGGER_AT_ONCE, 0, &req.pRSmaParam.qmsg2, &req.pRSmaParam.qmsg2Len,
+      if (mndConvertRsmaTask(&req.pRSmaParam.qmsg2, &req.pRSmaParam.qmsg2Len, pStb->pAst2, pStb->uid, STREAM_TRIGGER_AT_ONCE, 0,
                              req.pRSmaParam.xFilesFactor) != TSDB_CODE_SUCCESS) {
         return NULL;
       }

source/dnode/mnode/impl/src/mndStream.c

@@ -252,8 +252,12 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast
   }
 
   if (qExtractResultSchema(pAst, (int32_t *)&pStream->outputSchema.nCols, &pStream->outputSchema.pSchema) != 0) {
+    nodesDestroyNode(pAst);
     return -1;
   }
+  // free
+  nodesDestroyNode(pAst);
+
 
 #if 0
   printf("|");
@@ -269,7 +273,7 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast
     return -1;
   }
 
-  if (mndScheduleStream1(pMnode, pTrans, pStream) < 0) {
+  if (mndScheduleStream(pMnode, pTrans, pStream) < 0) {
     mError("stream:%ld, schedule stream since %s", pStream->uid, terrstr());
     return -1;
   }

source/dnode/mnode/impl/src/mndSync.c

@@ -61,6 +61,12 @@ void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbM
     }
     tsem_post(&pMgmt->syncSem);
   } else {
+    STrans *pTrans = mndAcquireTrans(pMnode, transId);
+    if (pTrans != NULL) {
+      mndTransExecute(pMnode, pTrans);
+      mndReleaseTrans(pMnode, pTrans);
+    }
+
     if (cbMeta.index - sdbGetApplyIndex(pMnode->pSdb) > 100) {
       SSnapshotMeta sMeta = {0};
       if (syncGetSnapshotMeta(pMnode->syncMgmt.sync, &sMeta) == 0) {
@@ -200,14 +206,14 @@ int32_t mndInitSync(SMnode *pMnode) {
     return -1;
   }
 
-  mDebug("mnode sync is opened, id:%" PRId64, pMgmt->sync);
+  mDebug("mnode-sync is opened, id:%" PRId64, pMgmt->sync);
   return 0;
 }
 
 void mndCleanupSync(SMnode *pMnode) {
   SSyncMgmt *pMgmt = &pMnode->syncMgmt;
   syncStop(pMgmt->sync);
-  mDebug("mnode sync is stopped, id:%" PRId64, pMgmt->sync);
+  mDebug("mnode-sync is stopped, id:%" PRId64, pMgmt->sync);
 
   tsem_destroy(&pMgmt->syncSem);
   memset(pMgmt, 0, sizeof(SSyncMgmt));

source/dnode/mnode/impl/src/mndTrans.c

@@ -52,8 +52,8 @@ static bool    mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans);
 static bool    mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans);
 static bool    mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans);
 static bool    mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans);
+static bool    mndCantExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsMaster(pMnode); }
 
-static void    mndTransExecute(SMnode *pMnode, STrans *pTrans);
 static void    mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans);
 static int32_t mndProcessTransReq(SRpcMsg *pReq);
 static int32_t mndProcessKillTransReq(SRpcMsg *pReq);
@@ -297,7 +297,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
       SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER)
       action.pRaw = taosMemoryMalloc(dataLen);
       if (action.pRaw == NULL) goto _OVER;
-      mTrace("raw:%p, is created", pData);
+      // mTrace("raw:%p, is created", pData);
       SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER);
       if (taosArrayPush(pTrans->redoActions, &action) == NULL) goto _OVER;
       action.pRaw = NULL;
@@ -330,7 +330,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
       SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER)
       action.pRaw = taosMemoryMalloc(dataLen);
       if (action.pRaw == NULL) goto _OVER;
-      mTrace("raw:%p, is created", pData);
+      // mTrace("raw:%p, is created", action.pRaw);
       SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER);
       if (taosArrayPush(pTrans->undoActions, &action) == NULL) goto _OVER;
       action.pRaw = NULL;
@@ -363,7 +363,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
       SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER)
       action.pRaw = taosMemoryMalloc(dataLen);
       if (action.pRaw == NULL) goto _OVER;
-      mTrace("raw:%p, is created", action.pRaw);
+      // mTrace("raw:%p, is created", action.pRaw);
       SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER);
       if (taosArrayPush(pTrans->commitActions, &action) == NULL) goto _OVER;
       action.pRaw = NULL;
@@ -517,12 +517,12 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) {
 
   if (pOld->stage == TRN_STAGE_COMMIT) {
     pOld->stage = TRN_STAGE_COMMIT_ACTION;
-    mTrace("trans:%d, stage from commit to commitAction", pNew->id);
+    mTrace("trans:%d, stage from commit to commitAction since perform update action", pNew->id);
   }
 
   if (pOld->stage == TRN_STAGE_ROLLBACK) {
     pOld->stage = TRN_STAGE_FINISHED;
-    mTrace("trans:%d, stage from rollback to finished", pNew->id);
+    mTrace("trans:%d, stage from rollback to finished since perform update action", pNew->id);
   }
   return 0;
 }
@@ -914,7 +914,7 @@ static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransActi
 
 static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction) {
   if (pAction->msgSent) return 0;
-  if (!pMnode->deploy && !mndIsMaster(pMnode)) return -1;
+  if (mndCantExecuteTransAction(pMnode)) return -1;
 
   int64_t signature = pTrans->id;
   signature = (signature << 32);
@@ -1114,9 +1114,9 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans)
       pTrans->lastEpset = pAction->epSet;
     }
 
-    if (code == 0) {
-      if (!pMnode->deploy && !mndIsMaster(pMnode)) break;
+    if (mndCantExecuteTransAction(pMnode)) break;
 
+    if (code == 0) {
       pTrans->code = 0;
       pTrans->redoActionPos++;
       mDebug("trans:%d, %s:%d is executed and need sync to other mnodes", pTrans->id, mndTransStr(pAction->stage),
@@ -1160,6 +1160,8 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans) {
     code = mndTransExecuteRedoActions(pMnode, pTrans);
   }
 
+  if (mndCantExecuteTransAction(pMnode)) return false;
+
   if (code == 0) {
     pTrans->code = 0;
     pTrans->stage = TRN_STAGE_COMMIT;
@@ -1185,6 +1187,8 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans) {
 }
 
 static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) {
+  if (mndCantExecuteTransAction(pMnode)) return false;
+
   bool    continueExec = true;
   int32_t code = mndTransCommit(pMnode, pTrans);
 
@@ -1233,6 +1237,8 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans) {
   bool    continueExec = true;
   int32_t code = mndTransExecuteUndoActions(pMnode, pTrans);
 
+  if (mndCantExecuteTransAction(pMnode)) return false;
+
   if (code == 0) {
     pTrans->stage = TRN_STAGE_ROLLBACK;
     mDebug("trans:%d, stage from undoAction to rollback", pTrans->id);
@@ -1250,6 +1256,8 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans) {
 }
 
 static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) {
+  if (mndCantExecuteTransAction(pMnode)) return false;
+
   bool    continueExec = true;
   int32_t code = mndTransRollback(pMnode, pTrans);
 
@@ -1284,10 +1292,11 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) {
   return continueExec;
 }
 
-static void mndTransExecute(SMnode *pMnode, STrans *pTrans) {
+void mndTransExecute(SMnode *pMnode, STrans *pTrans) {
   bool continueExec = true;
 
   while (continueExec) {
+    mDebug("trans:%d, continue to execute, stage:%s", pTrans->id, mndTransStr(pTrans->stage));
     pTrans->lastExecTime = taosGetTimestampMs();
     switch (pTrans->stage) {
       case TRN_STAGE_PREPARE:

source/dnode/mnode/sdb/src/sdb.c

@@ -163,7 +163,12 @@ void sdbSetApplyIndex(SSdb *pSdb, int64_t index) { pSdb->curVer = index; }
 
 void sdbSetApplyTerm(SSdb *pSdb, int64_t term) { pSdb->curTerm = term; }
 
-void sdbSetCurConfig(SSdb *pSdb, int64_t config) { pSdb->curConfig = config; }
+void sdbSetCurConfig(SSdb *pSdb, int64_t config) {
+  if (pSdb->curConfig != config) {
+    mDebug("mnode sync config set from %" PRId64 " to %" PRId64, pSdb->curConfig, config);
+    pSdb->curConfig = config;
+  }
+}
 
 int64_t sdbGetApplyIndex(SSdb *pSdb) { return pSdb->curVer; }
 

source/dnode/mnode/sdb/src/sdbFile.c

@@ -432,8 +432,8 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
   } else {
     pSdb->lastCommitVer = pSdb->curVer;
     pSdb->lastCommitTerm = pSdb->curTerm;
-    mDebug("write sdb file successfully, ver:%" PRId64 " term:%" PRId64 " file:%s", pSdb->lastCommitVer,
-           pSdb->lastCommitTerm, curfile);
+    mDebug("write sdb file successfully, index:%" PRId64 " term:%" PRId64 " config:%" PRId64 " file:%s",
+           pSdb->lastCommitVer, pSdb->lastCommitTerm, pSdb->curConfig, curfile);
   }
 
   terrno = code;

source/dnode/vnode/CMakeLists.txt

@@ -28,7 +28,6 @@ target_sources(
 
     # sma
     "src/sma/sma.c"
-    "src/sma/smaTDBImpl.c"
     "src/sma/smaEnv.c"
     "src/sma/smaOpen.c"
     "src/sma/smaRollup.c"

source/dnode/vnode/src/inc/sma.h

@@ -43,35 +43,17 @@ typedef struct SRSmaInfo    SRSmaInfo;
 struct SSmaEnv {
   TdThreadRwlock lock;
   int8_t         type;
-  TXN            txn;
-  void          *pPool;  // SPoolMem
-  SDiskID        did;
-  TDB           *dbEnv;  // TODO: If it's better to put it in smaIndex level?
-  char          *path;   // relative path
   SSmaStat      *pStat;
 };
 
 #define SMA_ENV_LOCK(env)       ((env)->lock)
 #define SMA_ENV_TYPE(env)       ((env)->type)
-#define SMA_ENV_DID(env)        ((env)->did)
-#define SMA_ENV_ENV(env)        ((env)->dbEnv)
-#define SMA_ENV_PATH(env)       ((env)->path)
 #define SMA_ENV_STAT(env)       ((env)->pStat)
 #define SMA_ENV_STAT_ITEMS(env) ((env)->pStat->smaStatItems)
 
 struct SSmaStatItem {
-  /**
-   * @brief The field 'state' is here to demonstrate if one smaIndex is ready to provide service.
-   *    - TSDB_SMA_STAT_OK: 1) The sma calculation of history data is finished; 2) Or recevied information from
-   * Streaming Module or TSDB local persistence.
-   *    - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open,
-   * without information about its previous state.
-   *    - TSDB_SMA_STAT_DROPPED: 1)sma dropped
-   * N.B. only applicable to tsma
-   */
-  int8_t    state;           // ETsdbSmaStat
-  SHashObj *expiredWindows;  // key: skey of time window, value: version
-  STSma    *pTSma;           // cache schema
+  int8_t state;  // ETsdbSmaStat
+  STSma *pTSma;  // cache schema
 };
 
 struct SSmaStat {
@@ -84,29 +66,6 @@ struct SSmaStat {
 #define SMA_STAT_ITEMS(s)     ((s)->smaStatItems)
 #define SMA_STAT_INFO_HASH(s) ((s)->rsmaInfoHash)
 
-struct SSmaKey {
-  TSKEY   skey;
-  int64_t groupId;
-};
-
-typedef struct SDBFile SDBFile;
-
-struct SDBFile {
-  int32_t fid;
-  TTB    *pDB;
-  char   *path;
-};
-
-int32_t tdSmaBeginCommit(SSmaEnv *pEnv);
-int32_t tdSmaEndCommit(SSmaEnv *pEnv);
-
-int32_t smaOpenDBEnv(TDB **ppEnv, const char *path);
-int32_t smaCloseDBEnv(TDB *pEnv);
-int32_t smaOpenDBF(TDB *pEnv, SDBFile *pDBF);
-int32_t smaCloseDBF(SDBFile *pDBF);
-int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn);
-void   *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen);
-
 void  tdDestroySmaEnv(SSmaEnv *pSmaEnv);
 void *tdFreeSmaEnv(SSmaEnv *pSmaEnv);
 #if 0
@@ -114,13 +73,6 @@ int32_t tbGetTSmaStatus(SSma *pSma, STSma *param, void *result);
 int32_t tbRemoveTSmaData(SSma *pSma, STSma *param, STimeWindow *pWin);
 #endif
 
-static FORCE_INLINE int32_t tdEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) {
-  int32_t len = 0;
-  len += taosEncodeFixedI64(pData, tsKey);
-  len += taosEncodeFixedI64(pData, groupId);
-  return len;
-}
-
 int32_t tdInitSma(SSma *pSma);
 int32_t tdDropTSma(SSma *pSma, char *pMsg);
 int32_t tdDropTSmaData(SSma *pSma, int64_t indexUid);
@@ -128,13 +80,11 @@ int32_t tdInsertRSmaData(SSma *pSma, char *msg);
 
 int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat);
 int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat);
-int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType);
+int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType, bool onlyCheck);
 
 int32_t tdLockSma(SSma *pSma);
 int32_t tdUnLockSma(SSma *pSma);
 
-int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg);
-
 static FORCE_INLINE int16_t tdTSmaAdd(SSma *pSma, int16_t n) { return atomic_add_fetch_16(&SMA_TSMA_NUM(pSma), n); }
 static FORCE_INLINE int16_t tdTSmaSub(SSma *pSma, int16_t n) { return atomic_sub_fetch_16(&SMA_TSMA_NUM(pSma), n); }
 
@@ -219,11 +169,8 @@ static int32_t  tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SDisk
 void *tdFreeRSmaInfo(SRSmaInfo *pInfo);
 
 int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg);
-int32_t tdUpdateExpiredWindowImpl(SSma *pSma, const SSubmitReq *pMsg, int64_t version);
-// TODO: This is the basic params, and should wrap the params to a queryHandle.
-int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult);
-
-int32_t tdGetTSmaDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
+int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg);
+int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/inc/vnodeInt.h

@@ -150,7 +150,6 @@ int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg);
 int32_t smaOpen(SVnode* pVnode);
 int32_t smaClose(SSma* pSma);
 
-int32_t tdUpdateExpireWindow(SSma* pSma, const SSubmitReq* pMsg, int64_t version);
 int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg);
 int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg);
 
@@ -227,7 +226,7 @@ struct SVnode {
   SQHandle*  pQuery;
 };
 
-#define TD_VID(PVNODE) (PVNODE)->config.vgId
+#define TD_VID(PVNODE) ((PVNODE)->config.vgId)
 
 #define VND_TSDB(vnd)       ((vnd)->pTsdb)
 #define VND_RSMA0(vnd)      ((vnd)->pTsdb)

source/dnode/vnode/src/meta/metaSma.c

@@ -34,13 +34,13 @@ int32_t metaCreateTSma(SMeta *pMeta, int64_t version, SSmaCfg *pCfg) {
   SMetaReader mr = {0};
 
   // validate req
+  // save smaIndex
   metaReaderInit(&mr, pMeta, 0);
   if (metaGetTableEntryByUid(&mr, pCfg->indexUid) == 0) {
-// TODO: just for pass case
 #if 1
-    terrno = TSDB_CODE_TDB_TSMA_ALREADY_EXIST;
+    terrno = TSDB_CODE_TSMA_ALREADY_EXIST;
     metaReaderClear(&mr);
-    return -1;
+    return -1;  // don't goto _err;
 #else
     metaReaderClear(&mr);
     return 0;

source/dnode/vnode/src/sma/sma.c

@@ -36,25 +36,9 @@ int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg) {
   return code;
 }
 
-int32_t tdUpdateExpireWindow(SSma* pSma, const SSubmitReq* pMsg, int64_t version) {
-  int32_t code = TSDB_CODE_SUCCESS;
-  if ((code = tdUpdateExpiredWindowImpl(pSma, pMsg, version)) < 0) {
-    smaWarn("vgId:%d, update expired sma window failed since %s", SMA_VID(pSma), tstrerror(terrno));
-  }
-  return code;
-}
-
-int32_t tdGetTSmaData(SSma* pSma, char* pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) {
-  int32_t code = TSDB_CODE_SUCCESS;
-  if ((code = tdGetTSmaDataImpl(pSma, pData, indexUid, querySKey, nMaxResult)) < 0) {
-    smaWarn("vgId:%d, get tsma data failed since %s", SMA_VID(pSma), tstrerror(terrno));
-  }
-  return code;
-}
-
 int32_t smaGetTSmaDays(SVnodeCfg* pCfg, void* pCont, uint32_t contLen, int32_t* days) {
   int32_t code = TSDB_CODE_SUCCESS;
-  if ((code = tdGetTSmaDaysImpl(pCfg, pCont, contLen, days)) < 0) {
+  if ((code = tdProcessTSmaGetDaysImpl(pCfg, pCont, contLen, days)) < 0) {
     smaWarn("vgId:%d, get tsma days failed since %s", pCfg->vgId, tstrerror(terrno));
   }
   smaDebug("vgId:%d, get tsma days %d", pCfg->vgId, *days);

source/dnode/vnode/src/sma/smaEnv.c

@@ -151,31 +151,11 @@ static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path,
     return NULL;
   }
 
-  ASSERT(path && (strlen(path) > 0));
-  SMA_ENV_PATH(pEnv) = strdup(path);
-  if (!SMA_ENV_PATH(pEnv)) {
-    tdFreeSmaEnv(pEnv);
-    return NULL;
-  }
-
-  SMA_ENV_DID(pEnv) = did;
-
   if (tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType) != TSDB_CODE_SUCCESS) {
     tdFreeSmaEnv(pEnv);
     return NULL;
   }
 
-  char aname[TSDB_FILENAME_LEN] = {0};
-  tfsAbsoluteName(SMA_TFS(pSma), did, path, aname);
-  if (smaOpenDBEnv(&pEnv->dbEnv, aname) != TSDB_CODE_SUCCESS) {
-    tdFreeSmaEnv(pEnv);
-    return NULL;
-  }
-
-  if (!(pEnv->pPool = openPool())) {
-    tdFreeSmaEnv(pEnv);
-    return NULL;
-  }
 
   return pEnv;
 }
@@ -205,10 +185,7 @@ void tdDestroySmaEnv(SSmaEnv *pSmaEnv) {
   if (pSmaEnv) {
     tdDestroySmaState(pSmaEnv->pStat, SMA_ENV_TYPE(pSmaEnv));
     taosMemoryFreeClear(pSmaEnv->pStat);
-    taosMemoryFreeClear(pSmaEnv->path);
     taosThreadRwlockDestroy(&(pSmaEnv->lock));
-    smaCloseDBEnv(pSmaEnv->dbEnv);
-    closePool(pSmaEnv->pPool);
   }
 }
 
@@ -242,7 +219,7 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) {
   }
 
   /**
-   *  1. Lazy mode utilized when init SSmaStat to update expired window(or hungry mode when tdNew).
+   *  1. Lazy mode utilized when init SSmaStat to update expire window(or hungry mode when tdNew).
    *  2. Currently, there is mutex lock when init SSmaEnv, thus no need add lock on SSmaStat, and please add lock if
    * tdInitSmaStat invoked in other multithread environment later.
    */
@@ -280,7 +257,6 @@ void *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem) {
   if (pSmaStatItem) {
     tDestroyTSma(pSmaStatItem->pTSma);
     taosMemoryFreeClear(pSmaStatItem->pTSma);
-    taosHashCleanup(pSmaStatItem->expiredWindows);
     taosMemoryFreeClear(pSmaStatItem);
   }
   return NULL;
@@ -341,7 +317,7 @@ int32_t tdUnLockSma(SSma *pSma) {
   return 0;
 }
 
-int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) {
+int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType, bool onlyCheck) {
   SSmaEnv *pEnv = NULL;
 
   // return if already init
@@ -399,63 +375,3 @@ int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) {
 
   return TSDB_CODE_SUCCESS;
 };
-
-int32_t tdSmaBeginCommit(SSmaEnv *pEnv) {
-  TXN *pTxn = &pEnv->txn;
-  // start a new txn
-  tdbTxnOpen(pTxn, 0, poolMalloc, poolFree, pEnv->pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-  if (tdbBegin(pEnv->dbEnv, pTxn) != 0) {
-    smaWarn("tdSma tdb begin commit fail");
-    return -1;
-  }
-  return 0;
-}
-
-int32_t tdSmaEndCommit(SSmaEnv *pEnv) {
-  TXN *pTxn = &pEnv->txn;
-
-  // Commit current txn
-  if (tdbCommit(pEnv->dbEnv, pTxn) != 0) {
-    smaWarn("tdSma tdb end commit fail");
-    return -1;
-  }
-  tdbTxnClose(pTxn);
-  clearPool(pEnv->pPool);
-  return 0;
-}
-
-#if 0
-/**
- * @brief Get the start TS key of the last data block of one interval/sliding.
- *
- * @param pSma
- * @param param
- * @param result
- * @return int32_t
- *         1) Return 0 and fill the result if the check procedure is normal;
- *         2) Return -1 if error occurs during the check procedure.
- */
-int32_t tdGetTSmaStatus(SSma *pSma, void *smaIndex, void *result) {
-  const char *procedure = "";
-  if (strncmp(procedure, "get the start TS key of the last data block", 100) != 0) {
-    return -1;
-  }
-  // fill the result
-  return TSDB_CODE_SUCCESS;
-}
-
-/**
- * @brief Remove the tSma data files related to param between pWin.
- *
- * @param pSma
- * @param param
- * @param pWin
- * @return int32_t
- */
-int32_t tdRemoveTSmaData(SSma *pSma, void *smaIndex, STimeWindow *pWin) {
-  // for ("tSmaFiles of param-interval-sliding between pWin") {
-  //   // remove the tSmaFile
-  // }
-  return TSDB_CODE_SUCCESS;
-}
-#endif

source/dnode/vnode/src/sma/smaRollup.c

@@ -65,7 +65,7 @@ static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SA
   pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), suid, sizeof(tb_uid_t));
   if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
     smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid);
-    terrno = TSDB_CODE_TDB_INVALID_SMA_STAT;
+    terrno = TSDB_CODE_RSMA_INVALID_STAT;
     return TSDB_CODE_FAILED;
   }
 
@@ -132,7 +132,7 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui
   SSmaStat *pStat = SMA_ENV_STAT(pEnv);
   SHashObj *infoHash = NULL;
   if (!pStat || !(infoHash = SMA_STAT_INFO_HASH(pStat))) {
-    terrno = TSDB_CODE_TDB_INVALID_SMA_STAT;
+    terrno = TSDB_CODE_RSMA_INVALID_STAT;
     return TSDB_CODE_FAILED;
   }
 
@@ -167,13 +167,13 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui
  */
 int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) {
   SSma *pSma = pVnode->pSma;
-  SMeta *pMeta = pVnode->pMeta;
-  SMsgCb *pMsgCb = &pVnode->msgCb;
   if (!pReq->rollup) {
     smaTrace("vgId:%d, return directly since no rollup for stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid);
     return TSDB_CODE_SUCCESS;
   }
 
+  SMeta *pMeta = pVnode->pMeta;
+  SMsgCb *pMsgCb = &pVnode->msgCb;
   SRSmaParam *param = &pReq->pRSmaParam;
 
   if ((param->qmsg1Len == 0) && (param->qmsg2Len == 0)) {
@@ -181,7 +181,7 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) {
     return TSDB_CODE_SUCCESS;
   }
 
-  if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_ROLLUP) != TSDB_CODE_SUCCESS) {
+  if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_ROLLUP, false) != TSDB_CODE_SUCCESS) {
     terrno = TSDB_CODE_TDB_INIT_FAILED;
     return TSDB_CODE_FAILED;
   }

source/dnode/vnode/src/sma/smaTDBImpl.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define ALLOW_FORBID_FUNC
-
-#include "sma.h"
-
-int32_t smaOpenDBEnv(TDB **ppEnv, const char *path) {
-  int ret = 0;
-
-  if (path == NULL) return -1;
-
-  ret = tdbOpen(path, 4096, 256, ppEnv);  // use as param
-
-  if (ret != 0) {
-    smaError("failed to create tsdb db env, ret = %d", ret);
-    return -1;
-  }
-
-  return 0;
-}
-
-int32_t smaCloseDBEnv(TDB *pEnv) { return tdbClose(pEnv); }
-
-static inline int tdSmaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) {
-  const SSmaKey *pKey1 = (const SSmaKey *)arg1;
-  const SSmaKey *pKey2 = (const SSmaKey *)arg2;
-
-  ASSERT(len1 == len2 && len1 == sizeof(SSmaKey));
-
-  if (pKey1->skey < pKey2->skey) {
-    return -1;
-  } else if (pKey1->skey > pKey2->skey) {
-    return 1;
-  }
-  if (pKey1->groupId < pKey2->groupId) {
-    return -1;
-  } else if (pKey1->groupId > pKey2->groupId) {
-    return 1;
-  }
-
-  return 0;
-}
-
-static int32_t smaOpenDBDb(TTB **ppDB, TDB *pEnv, const char *pFName) {
-  tdb_cmpr_fn_t compFunc;
-
-  // Create a database
-  compFunc = tdSmaKeyCmpr;
-  if (tdbTbOpen(pFName, -1, -1, compFunc, pEnv, ppDB) < 0) {
-    return -1;
-  }
-
-  return 0;
-}
-
-static int32_t smaCloseDBDb(TTB *pDB) { return tdbTbClose(pDB); }
-
-int32_t smaOpenDBF(TDB *pEnv, SDBFile *pDBF) {
-  // TEnv is shared by a group of SDBFile
-  if (!pEnv || !pDBF) {
-    terrno = TSDB_CODE_INVALID_PTR;
-    return -1;
-  }
-
-  // Open DBF
-  if (smaOpenDBDb(&(pDBF->pDB), pEnv, pDBF->path) < 0) {
-    smaError("failed to open DBF: %s", pDBF->path);
-    smaCloseDBDb(pDBF->pDB);
-    return -1;
-  }
-
-  return 0;
-}
-
-int32_t smaCloseDBF(SDBFile *pDBF) {
-  int32_t ret = 0;
-  if (pDBF->pDB) {
-    ret = smaCloseDBDb(pDBF->pDB);
-    pDBF->pDB = NULL;
-  }
-  taosMemoryFreeClear(pDBF->path);
-  return ret;
-}
-
-int32_t smaSaveSmaToDB(SDBFile *pDBF, void *pKey, int32_t keyLen, void *pVal, int32_t valLen, TXN *txn) {
-  int32_t ret;
-
-  printf("save tsma data into %s, keyLen:%d valLen:%d txn:%p\n", pDBF->path, keyLen, valLen, txn);
-  ret = tdbTbUpsert(pDBF->pDB, pKey, keyLen, pVal, valLen, txn);
-  if (ret < 0) {
-    smaError("failed to upsert tsma data into db, ret = %d", ret);
-    return -1;
-  }
-
-  return 0;
-}
-
-void *smaGetSmaDataByKey(SDBFile *pDBF, const void *pKey, int32_t keyLen, int32_t *valLen) {
-  void *pVal = NULL;
-  int   ret;
-
-  ret = tdbTbGet(pDBF->pDB, pKey, keyLen, &pVal, valLen);
-
-  if (ret < 0) {
-    smaError("failed to get tsma data from db, ret = %d", ret);
-    return NULL;
-  }
-
-  ASSERT(*valLen >= 0);
-
-  // TODO: lock?
-  // TODO: Would the key/value be destoryed during return the data?
-  // TODO: How about the key is updated while value length is changed? The original value buffer would be freed
-  // automatically?
-
-  return pVal;
-}
\ No newline at end of file

source/dnode/vnode/src/sma/smaTimeRange.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "sma.h"
-#include "tsdb.h"
-
-typedef STsdbCfg STSmaKeepCfg;
-
-#undef _TEST_SMA_PRINT_DEBUG_LOG_
-#define SMA_STORAGE_TSDB_MINUTES 86400
-#define SMA_STORAGE_TSDB_TIMES   10
-#define SMA_STORAGE_SPLIT_FACTOR 144  // least records in tsma file
-#define SMA_KEY_LEN              16   // TSKEY+groupId 8+8
-#define SMA_DROP_EXPIRED_TIME    10   // default is 10 seconds
-
-#define SMA_STATE_ITEM_HASH_SLOT 32
-
-typedef struct {
-  SSma         *pSma;
-  SDBFile       dFile;
-  const SArray *pDataBlocks;  // sma data
-  int64_t       interval;     // interval with the precision of DB
-} STSmaWriteH;
-
-typedef struct {
-  int32_t iter;
-  int32_t fid;
-} SmaFsIter;
-
-typedef struct {
-  STsdb    *pTsdb;
-  SSma     *pSma;
-  SDBFile   dFile;
-  int64_t   interval;   // interval with the precision of DB
-  int32_t   blockSize;  // size of SMA block item
-  int32_t   days;
-  int8_t    storageLevel;
-  SmaFsIter smaFsIter;
-} STSmaReadH;
-
-typedef enum {
-  SMA_STORAGE_LEVEL_TSDB = 0,     // use days of self-defined  e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f200.tsma
-  SMA_STORAGE_LEVEL_DFILESET = 1  // use days of TS data       e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma
-} ESmaStorageLevel;
-
-// static func
-
-static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted);
-static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval);
-static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval,
-                                int8_t intervalUnit);
-static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit);
-static void    tdDestroyTSmaWriteH(STSmaWriteH *pSmaH);
-static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel);
-static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid);
-static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey);
-static bool    tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey);
-static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen,
-                                  TXN *txn);
-// expired window
-
-static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, int64_t version);
-static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey);
-static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid);
-
-// read data
-
-// implementation
-
-/**
- * @brief
- *
- * @param pSmaH
- * @param pSma
- * @param interval
- * @param intervalUnit
- * @return int32_t
- */
-static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit) {
-  STSmaKeepCfg *pCfg = SMA_TSDB_CFG(pSma);
-  pSmaH->pSma = pSma;
-  pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true);
-  pSmaH->storageLevel = tdGetSmaStorageLevel(pCfg, interval);
-  pSmaH->days = tdGetTSmaDays(pSma, pSmaH->interval, pSmaH->storageLevel);
-  return TSDB_CODE_SUCCESS;
-}
-
-/**
- * @brief Init of tSma FS
- *
- * @param pReadH
- * @param indexUid
- * @param skey
- * @return int32_t
- */
-static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey) {
-  SSma *pSma = pSmaH->pSma;
-
-  int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, SMA_TSDB_CFG(pSma)->precision));
-  char    tSmaFile[TSDB_FILENAME_LEN] = {0};
-  snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, SMA_VID(pSma), fid);
-  pSmaH->dFile.path = strdup(tSmaFile);
-  pSmaH->smaFsIter.iter = 0;
-  pSmaH->smaFsIter.fid = fid;
-  return TSDB_CODE_SUCCESS;
-}
-
-/**
- * @brief Set and open tSma file if it has key locates in queryWin.
- *
- * @param pReadH
- * @param param
- * @param queryWin
- * @return true
- * @return false
- */
-static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) {
-  // SArray *smaFs = pReadH->pTsdb->fs->cstatus->sf;
-  // int32_t nSmaFs = taosArrayGetSize(smaFs);
-
-  smaCloseDBF(&pReadH->dFile);
-
-#if 0
-  while (pReadH->smaFsIter.iter < nSmaFs) {
-    void *pSmaFile = taosArrayGet(smaFs, pReadH->smaFsIter.iter);
-    if (pSmaFile) {  // match(indexName, queryWindow)
-      // TODO: select the file by index_name ...
-      pReadH->dFile = pSmaFile;
-      ++pReadH->smaFsIter.iter;
-      break;
-    }
-    ++pReadH->smaFsIter.iter;
-  }
-
-  if (pReadH->pDFile) {
-    tdDebug("vg%d: smaFile %s matched", REPO_ID(pReadH->pTsdb), "[pSmaFile dir]");
-    return true;
-  }
-#endif
-
-  return false;
-}
-
-/**
- * @brief Approximate value for week/month/year.
- *
- * @param interval
- * @param intervalUnit
- * @param precision
- * @param adjusted Interval already adjusted according to DB precision
- * @return int64_t
- */
-static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted) {
-  if (adjusted) {
-    return interval;
-  }
-
-  switch (intervalUnit) {
-    case TIME_UNIT_YEAR:  // approximate value
-      interval *= 365 * 86400 * 1e3;
-      break;
-    case TIME_UNIT_MONTH:  // approximate value
-      interval *= 30 * 86400 * 1e3;
-      break;
-    case TIME_UNIT_WEEK:  // approximate value
-      interval *= 7 * 86400 * 1e3;
-      break;
-    case TIME_UNIT_DAY:  // the interval for tSma calculation must <= day
-      interval *= 86400 * 1e3;
-      break;
-    case TIME_UNIT_HOUR:
-      interval *= 3600 * 1e3;
-      break;
-    case TIME_UNIT_MINUTE:
-      interval *= 60 * 1e3;
-      break;
-    case TIME_UNIT_SECOND:
-      interval *= 1e3;
-      break;
-    default:
-      break;
-  }
-
-  switch (precision) {
-    case TSDB_TIME_PRECISION_MILLI:
-      if (TIME_UNIT_MICROSECOND == intervalUnit) {  // us
-        return interval / 1e3;
-      } else if (TIME_UNIT_NANOSECOND == intervalUnit) {  //  nano second
-        return interval / 1e6;
-      } else {  // ms
-        return interval;
-      }
-      break;
-    case TSDB_TIME_PRECISION_MICRO:
-      if (TIME_UNIT_MICROSECOND == intervalUnit) {  // us
-        return interval;
-      } else if (TIME_UNIT_NANOSECOND == intervalUnit) {  //  ns
-        return interval / 1e3;
-      } else {  // ms
-        return interval * 1e3;
-      }
-      break;
-    case TSDB_TIME_PRECISION_NANO:
-      if (TIME_UNIT_MICROSECOND == intervalUnit) {  // us
-        return interval * 1e3;
-      } else if (TIME_UNIT_NANOSECOND == intervalUnit) {  // ns
-        return interval;
-      } else {  // ms
-        return interval * 1e6;
-      }
-      break;
-    default:                                        // ms
-      if (TIME_UNIT_MICROSECOND == intervalUnit) {  // us
-        return interval / 1e3;
-      } else if (TIME_UNIT_NANOSECOND == intervalUnit) {  //  ns
-        return interval / 1e6;
-      } else {  // ms
-        return interval;
-      }
-      break;
-  }
-  return interval;
-}
-
-static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval,
-                                int8_t intervalUnit) {
-  pSmaH->pSma = pSma;
-  pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true);
-  pSmaH->pDataBlocks = pDataBlocks;
-  pSmaH->dFile.fid = SMA_IVLD_FID;
-  return TSDB_CODE_SUCCESS;
-}
-
-static void tdDestroyTSmaWriteH(STSmaWriteH *pSmaH) {
-  if (pSmaH) {
-    smaCloseDBF(&pSmaH->dFile);
-  }
-}
-
-static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid) {
-  SSma *pSma = pSmaH->pSma;
-  ASSERT(!pSmaH->dFile.path && !pSmaH->dFile.pDB);
-
-  pSmaH->dFile.fid = fid;
-  char tSmaFile[TSDB_FILENAME_LEN] = {0};
-  snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, SMA_VID(pSma), fid);
-  pSmaH->dFile.path = strdup(tSmaFile);
-
-  return TSDB_CODE_SUCCESS;
-}
-
-/**
- * @brief
- *
- * @param pSma
- * @param interval Interval calculated by DB's precision
- * @param storageLevel
- * @return int32_t
- */
-static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel) {
-  STsdbCfg *pCfg = SMA_TSDB_CFG(pSma);
-  int32_t   daysPerFile = pCfg->days;  // unit is minute
-
-  if (storageLevel == SMA_STORAGE_LEVEL_TSDB) {
-    int32_t minutes = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerMin[pCfg->precision]);
-    if (minutes > SMA_STORAGE_TSDB_MINUTES) {
-      daysPerFile = SMA_STORAGE_TSDB_MINUTES;
-    }
-  }
-
-  return daysPerFile;
-}
-
-/**
- * @brief Judge the tSma storage level
- *
- * @param pCfg
- * @param interval
- * @return int32_t
- */
-static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval) {
-  int64_t mInterval = convertTimeFromPrecisionToUnit(interval, pCfg->precision, TIME_UNIT_MINUTE);
-  if (pCfg->days / mInterval >= SMA_STORAGE_SPLIT_FACTOR) {
-    return SMA_STORAGE_LEVEL_DFILESET;
-  }
-  return SMA_STORAGE_LEVEL_TSDB;
-}
-
-/**
- * @brief Insert/Update Time-range-wise SMA data.
- *  - If interval < SMA_STORAGE_SPLIT_HOURS(e.g. 24), save the SMA data as a part of DFileSet to e.g.
- * v3f1900.tsma.${sma_index_name}. The days is the same with that for TS data files.
- *  - If interval >= SMA_STORAGE_SPLIT_HOURS, save the SMA data to e.g. vnode3/tsma/v3f632.tsma.${sma_index_name}. The
- * days is 30 times of the interval, and the minimum days is SMA_STORAGE_TSDB_DAYS(30d).
- *  - The destination file of one data block for some interval is determined by its start TS key.
- *
- * @param pSma
- * @param msg
- * @return int32_t
- */
-int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
-  STsdbCfg     *pCfg = SMA_TSDB_CFG(pSma);
-  const SArray *pDataBlocks = (const SArray *)msg;
-  int64_t       testSkey = TSKEY_INITIAL_VAL;
-
-  // TODO: destroy SSDataBlocks(msg)
-
-  // For super table aggregation, the sma data is stored in vgroup calculated from the hash value of stable name. Thus
-  // the sma data would arrive ahead of the update-expired-window msg.
-  if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) != TSDB_CODE_SUCCESS) {
-    terrno = TSDB_CODE_TDB_INIT_FAILED;
-    return TSDB_CODE_FAILED;
-  }
-
-  if (!pDataBlocks) {
-    terrno = TSDB_CODE_INVALID_PTR;
-    smaWarn("vgId:%d, insert tSma data failed since pDataBlocks is NULL", SMA_VID(pSma));
-    return terrno;
-  }
-
-  if (taosArrayGetSize(pDataBlocks) <= 0) {
-    terrno = TSDB_CODE_INVALID_PARA;
-    smaWarn("vgId:%d, insert tSma data failed since pDataBlocks is empty", SMA_VID(pSma));
-    return TSDB_CODE_FAILED;
-  }
-
-  SSmaEnv      *pEnv = SMA_TSMA_ENV(pSma);
-  SSmaStat     *pStat = SMA_ENV_STAT(pEnv);
-  SSmaStatItem *pItem = NULL;
-
-  tdRefSmaStat(pSma, pStat);
-
-  if (pStat && SMA_STAT_ITEMS(pStat)) {
-    pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid));
-  }
-
-  if (!pItem || !(pItem = *(SSmaStatItem **)pItem) || tdSmaStatIsDropped(pItem)) {
-    terrno = TSDB_CODE_TDB_INVALID_SMA_STAT;
-    tdUnRefSmaStat(pSma, pStat);
-    return TSDB_CODE_FAILED;
-  }
-
-  STSma      *pTSma = pItem->pTSma;
-  STSmaWriteH tSmaH = {0};
-
-  if (tdInitTSmaWriteH(&tSmaH, pSma, pDataBlocks, pTSma->interval, pTSma->intervalUnit) != 0) {
-    return TSDB_CODE_FAILED;
-  }
-
-  char rPath[TSDB_FILENAME_LEN] = {0};
-  char aPath[TSDB_FILENAME_LEN] = {0};
-  snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid);
-  tfsAbsoluteName(SMA_TFS(pSma), SMA_ENV_DID(pEnv), rPath, aPath);
-  if (!taosCheckExistFile(aPath)) {
-    if (tfsMkdirRecurAt(SMA_TFS(pSma), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) {
-      tdUnRefSmaStat(pSma, pStat);
-      return TSDB_CODE_FAILED;
-    }
-  }
-
-  // Step 1: Judge the storage level and days
-  int32_t storageLevel = tdGetSmaStorageLevel(pCfg, tSmaH.interval);
-  int32_t minutePerFile = tdGetTSmaDays(pSma, tSmaH.interval, storageLevel);
-
-  char    smaKey[SMA_KEY_LEN] = {0};  // key: skey + groupId
-  char    dataBuf[512] = {0};         // val: aggr data // TODO: handle 512 buffer?
-  void   *pDataBuf = NULL;
-  int32_t sz = taosArrayGetSize(pDataBlocks);
-  for (int32_t i = 0; i < sz; ++i) {
-    SSDataBlock *pDataBlock = taosArrayGet(pDataBlocks, i);
-    int32_t      colNum = pDataBlock->info.numOfCols;
-    int32_t      rows = pDataBlock->info.rows;
-    int32_t      rowSize = pDataBlock->info.rowSize;
-    int64_t      groupId = pDataBlock->info.groupId;
-    for (int32_t j = 0; j < rows; ++j) {
-      printf("|");
-      TSKEY skey = TSKEY_INITIAL_VAL;  //  the start key of TS window by interval
-      void *pSmaKey = &smaKey;
-      bool  isStartKey = false;
-
-      int32_t tlen = 0;     // reset the len
-      pDataBuf = &dataBuf;  // reset the buf
-      for (int32_t k = 0; k < colNum; ++k) {
-        SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
-        void            *var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
-        switch (pColInfoData->info.type) {
-          case TSDB_DATA_TYPE_TIMESTAMP:
-            if (!isStartKey) {
-              isStartKey = true;
-              skey = *(TSKEY *)var;
-              testSkey = skey;
-              printf("= skey %" PRIi64 " groupId = %" PRIi64 "|", skey, groupId);
-              tdEncodeTSmaKey(groupId, skey, &pSmaKey);
-            } else {
-              printf(" %" PRIi64 " |", *(int64_t *)var);
-              tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var);
-              break;
-            }
-            break;
-          case TSDB_DATA_TYPE_BOOL:
-          case TSDB_DATA_TYPE_UTINYINT:
-            printf(" %15d |", *(uint8_t *)var);
-            tlen += taosEncodeFixedU8(&pDataBuf, *(uint8_t *)var);
-            break;
-          case TSDB_DATA_TYPE_TINYINT:
-            printf(" %15d |", *(int8_t *)var);
-            tlen += taosEncodeFixedI8(&pDataBuf, *(int8_t *)var);
-            break;
-          case TSDB_DATA_TYPE_SMALLINT:
-            printf(" %15d |", *(int16_t *)var);
-            tlen += taosEncodeFixedI16(&pDataBuf, *(int16_t *)var);
-            break;
-          case TSDB_DATA_TYPE_USMALLINT:
-            printf(" %15d |", *(uint16_t *)var);
-            tlen += taosEncodeFixedU16(&pDataBuf, *(uint16_t *)var);
-            break;
-          case TSDB_DATA_TYPE_INT:
-            printf(" %15d |", *(int32_t *)var);
-            tlen += taosEncodeFixedI32(&pDataBuf, *(int32_t *)var);
-            break;
-          case TSDB_DATA_TYPE_FLOAT:
-            printf(" %15f |", *(float *)var);
-            tlen += taosEncodeBinary(&pDataBuf, var, sizeof(float));
-            break;
-          case TSDB_DATA_TYPE_UINT:
-            printf(" %15u |", *(uint32_t *)var);
-            tlen += taosEncodeFixedU32(&pDataBuf, *(uint32_t *)var);
-            break;
-          case TSDB_DATA_TYPE_BIGINT:
-            printf(" %15ld |", *(int64_t *)var);
-            tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var);
-            break;
-          case TSDB_DATA_TYPE_DOUBLE:
-            printf(" %15lf |", *(double *)var);
-            tlen += taosEncodeBinary(&pDataBuf, var, sizeof(double));
-          case TSDB_DATA_TYPE_UBIGINT:
-            printf(" %15lu |", *(uint64_t *)var);
-            tlen += taosEncodeFixedU64(&pDataBuf, *(uint64_t *)var);
-            break;
-          case TSDB_DATA_TYPE_NCHAR: {
-            char tmpChar[100] = {0};
-            strncpy(tmpChar, varDataVal(var), varDataLen(var));
-            printf(" %s |", tmpChar);
-            tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var));
-            break;
-          }
-          case TSDB_DATA_TYPE_VARCHAR: {  // TSDB_DATA_TYPE_BINARY
-            char tmpChar[100] = {0};
-            strncpy(tmpChar, varDataVal(var), varDataLen(var));
-            printf(" %s |", tmpChar);
-            tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var));
-            break;
-          }
-          case TSDB_DATA_TYPE_VARBINARY:
-            // TODO: add binary/varbinary
-            TASSERT(0);
-          default:
-            printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type);
-            TASSERT(0);
-            break;
-        }
-      }
-      printf("\n");
-      // if ((tlen > 0) && (skey != TSKEY_INITIAL_VAL)) {
-      if (tlen > 0) {
-        int32_t fid = (int32_t)(TSDB_KEY_FID(skey, minutePerFile, pCfg->precision));
-
-        // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index
-        // file
-        //         - Set and open the DFile or the B+Tree file
-        // TODO: tsdbStartTSmaCommit();
-        if (fid != tSmaH.dFile.fid) {
-          if (tSmaH.dFile.fid != SMA_IVLD_FID) {
-            tdSmaEndCommit(pEnv);
-            smaCloseDBF(&tSmaH.dFile);
-          }
-          tdSetTSmaDataFile(&tSmaH, indexUid, fid);
-          smaDebug("vgId:%d, write to DBF %s, days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi32
-                   " queryKey:%" PRIi64,
-                   SMA_VID(pSma), tSmaH.dFile.path, minutePerFile, tSmaH.interval, storageLevel, testSkey);
-          if (smaOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) {
-            smaWarn("vgId:%d, open DB file %s failed since %s", SMA_VID(pSma),
-                    tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno));
-            tdDestroyTSmaWriteH(&tSmaH);
-            tdUnRefSmaStat(pSma, pStat);
-            return TSDB_CODE_FAILED;
-          }
-          tdSmaBeginCommit(pEnv);
-        }
-
-        if (tdInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen, &pEnv->txn) != 0) {
-          smaWarn("vgId:%d, insert tsma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64
-                  " since %s",
-                  SMA_VID(pSma), indexUid, skey, groupId, tstrerror(terrno));
-          tdSmaEndCommit(pEnv);
-          tdDestroyTSmaWriteH(&tSmaH);
-          tdUnRefSmaStat(pSma, pStat);
-          return TSDB_CODE_FAILED;
-        }
-
-        smaDebug("vgId:%d, insert tsma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64,
-                 SMA_VID(pSma), indexUid, skey, groupId);
-        // TODO:tsdbEndTSmaCommit();
-
-        // Step 3: reset the SSmaStat
-        tdResetExpiredWindow(pSma, pStat, indexUid, skey);
-      } else {
-        smaWarn("vgId:%d, invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64,
-                SMA_VID(pSma), skey, tlen, indexUid);
-      }
-    }
-  }
-  tdSmaEndCommit(pEnv);  // TODO: not commit for every insert
-  tdDestroyTSmaWriteH(&tSmaH);
-  tdUnRefSmaStat(pSma, pStat);
-
-  return TSDB_CODE_SUCCESS;
-}
-
-int32_t tdDropTSmaData(SSma *pSma, int64_t indexUid) {
-  int32_t code = TSDB_CODE_SUCCESS;
-  if ((code = tdDropTSmaDataImpl(pSma, indexUid)) < 0) {
-    smaWarn("vgId:%d, drop tSma data failed since %s", SMA_VID(pSma), tstrerror(terrno));
-  }
-  return code;
-}
-
-/**
- * @brief Insert TSma data blocks to DB File build by B+Tree
- *
- * @param pSmaH
- * @param smaKey  tableUid-colId-skeyOfWindow(8-2-8)
- * @param keyLen
- * @param pData
- * @param dataLen
- * @return int32_t
- */
-static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen,
-                                  TXN *txn) {
-  SDBFile *pDBFile = &pSmaH->dFile;
-
-  // TODO: insert tsma data blocks into B+Tree(TTB)
-  if (smaSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) {
-    smaWarn("vgId:%d, insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail",
-            SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
-    return TSDB_CODE_FAILED;
-  }
-  smaDebug("vgId:%d, insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed",
-           SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
-
-#ifdef _TEST_SMA_PRINT_DEBUG_LOG_
-  uint32_t valueSize = 0;
-  void    *data = tdGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize);
-  ASSERT(data != NULL);
-  for (uint32_t v = 0; v < valueSize; v += 8) {
-    smaWarn("vgId:%d, insert sma data val[%d] %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v));
-  }
-#endif
-  return TSDB_CODE_SUCCESS;
-}
-
-/**
- * @brief When sma data received from stream computing, make the relative expired window valid.
- *
- * @param pSma
- * @param pStat
- * @param indexUid
- * @param skey
- * @return int32_t
- */
-static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey) {
-  SSmaStatItem *pItem = NULL;
-
-  tdRefSmaStat(pSma, pStat);
-
-  if (pStat && SMA_STAT_ITEMS(pStat)) {
-    pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid));
-  }
-  if ((pItem) && ((pItem = *(SSmaStatItem **)pItem))) {
-    // pItem resides in hash buffer all the time unless drop sma index
-    // TODO: multithread protect
-    if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) {
-      // error handling
-      tdUnRefSmaStat(pSma, pStat);
-      smaWarn("vgId:%d, remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma), skey,
-              indexUid);
-      return TSDB_CODE_FAILED;
-    }
-    smaDebug("vgId:%d, remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", SMA_VID(pSma),
-             skey, indexUid);
-    // TODO: use a standalone interface to received state upate notification from stream computing module.
-    /**
-     * @brief state
-     *  - When SMA env init in TSDB, its status is TSDB_SMA_STAT_OK.
-     *  - In startup phase of stream computing module, it should notify the SMA env in TSDB to expired if needed(e.g.
-     * when batch data caculation not finised)
-     *  - When TSDB_SMA_STAT_OK, the stream computing module should also notify that to the SMA env in TSDB.
-     */
-    pItem->state = TSDB_SMA_STAT_OK;
-  } else {
-    // error handling
-    tdUnRefSmaStat(pSma, pStat);
-    smaWarn("vgId:%d, expired window %" PRIi64 " not exists for sma index %" PRIi64, SMA_VID(pSma), skey, indexUid);
-    return TSDB_CODE_FAILED;
-  }
-
-  tdUnRefSmaStat(pSma, pStat);
-  return TSDB_CODE_SUCCESS;
-}
-
-/**
- * @brief Drop tSma data and local cache
- *        - insert/query reference
- * @param pSma
- * @param msg
- * @return int32_t
- */
-static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid) {
-  SSmaEnv *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma));
-
-  // clear local cache
-  if (pEnv) {
-    smaDebug("vgId:%d, drop tSma local cache for %" PRIi64, SMA_VID(pSma), indexUid);
-
-    SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid));
-    if ((pItem) || ((pItem = *(SSmaStatItem **)pItem))) {
-      if (tdSmaStatIsDropped(pItem)) {
-        smaDebug("vgId:%d, tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid);
-        return TSDB_CODE_TDB_INVALID_ACTION;  // TODO: duplicate drop msg would be intercepted by mnode
-      }
-
-      tdWLockSmaEnv(pEnv);
-      if (tdSmaStatIsDropped(pItem)) {
-        tdUnLockSmaEnv(pEnv);
-        smaDebug("vgId:%d, tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid);
-        return TSDB_CODE_TDB_INVALID_ACTION;  // TODO: duplicate drop msg would be intercepted by mnode
-      }
-      tdSmaStatSetDropped(pItem);
-      tdUnLockSmaEnv(pEnv);
-
-      int32_t nSleep = 0;
-      int32_t refVal = INT32_MAX;
-      while (true) {
-        if ((refVal = T_REF_VAL_GET(SMA_ENV_STAT(pEnv))) <= 0) {
-          smaDebug("vgId:%d, drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal);
-          break;
-        }
-        smaDebug("vgId:%d, wait 1s to drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal);
-        taosSsleep(1);
-        if (++nSleep > SMA_DROP_EXPIRED_TIME) {
-          smaDebug("vgId:%d, drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep, refVal);
-          break;
-        };
-      }
-
-      tdFreeSmaStatItem(pItem);
-      smaDebug("vgId:%d, getTSmaDataImpl failed since no index %" PRIi64 " in local cache", SMA_VID(pSma), indexUid);
-    }
-  }
-  // clear sma data files
-  // TODO:
-  return TSDB_CODE_SUCCESS;
-}
-
-/**
- * @brief
- *
- * @param pSma Return the data between queryWin and fill the pData.
- * @param pData
- * @param indexUid
- * @param pQuerySKey
- * @param nMaxResult The query invoker should control the nMaxResult need to return to avoid OOM.
- * @return int32_t
- */
-int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) {
-  SSmaEnv  *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma));
-  SSmaStat *pStat = NULL;
-
-  if (!pEnv) {
-    terrno = TSDB_CODE_INVALID_PTR;
-    smaWarn("vgId:%d, getTSmaDataImpl failed since pTSmaEnv is NULL", SMA_VID(pSma));
-    return TSDB_CODE_FAILED;
-  }
-
-  pStat = SMA_ENV_STAT(pEnv);
-
-  tdRefSmaStat(pSma, pStat);
-  SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid));
-  if (!pItem || !(pItem = *(SSmaStatItem **)pItem)) {
-    // Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if
-    // it's NULL.
-    tdUnRefSmaStat(pSma, pStat);
-    terrno = TSDB_CODE_TDB_INVALID_ACTION;
-    smaDebug("vgId:%d, getTSmaDataImpl failed since no index %" PRIi64, SMA_VID(pSma), indexUid);
-    return TSDB_CODE_FAILED;
-  }
-
-#if 0
-  int32_t nQueryWin = taosArrayGetSize(pQuerySKey);
-  for (int32_t n = 0; n < nQueryWin; ++n) {
-    TSKEY skey = taosArrayGet(pQuerySKey, n);
-    if (taosHashGet(pItem->expiredWindows, &skey, sizeof(TSKEY))) {
-      // TODO: mark this window as expired.
-    }
-  }
-#endif
-
-#if 1
-  int8_t smaStat = 0;
-  if (!tdSmaStatIsOK(pItem, &smaStat)) {  // TODO: multiple check for large scale sma query
-    tdUnRefSmaStat(pSma, pStat);
-    terrno = TSDB_CODE_TDB_INVALID_SMA_STAT;
-    smaWarn("vgId:%d, getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, SMA_VID(pSma), indexUid,
-            tstrerror(terrno), smaStat);
-    return TSDB_CODE_FAILED;
-  }
-
-  if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY))) {
-    // TODO: mark this window as expired.
-    smaDebug("vgId:%d, skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma), querySKey,
-             indexUid);
-  } else {
-    smaDebug("vgId:%d, skey %" PRIi64 " of window not in expired window for index %" PRIi64, SMA_VID(pSma), querySKey,
-             indexUid);
-  }
-
-  STSma *pTSma = pItem->pTSma;
-#endif
-
-#if 1
-  STSmaReadH tReadH = {0};
-  tdInitTSmaReadH(&tReadH, pSma, pTSma->interval, pTSma->intervalUnit);
-  smaCloseDBF(&tReadH.dFile);
-
-  tdUnRefSmaStat(pSma, pStat);
-
-  tdInitTSmaFile(&tReadH, indexUid, querySKey);
-  smaDebug("### vgId:%d read from DBF %s  days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi8 " queryKey:%" PRIi64,
-           SMA_VID(pSma), tReadH.dFile.path, tReadH.days, tReadH.interval, tReadH.storageLevel, querySKey);
-  if (smaOpenDBF(pEnv->dbEnv, &tReadH.dFile) != 0) {
-    smaWarn("vgId:%d, open DBF %s failed since %s", SMA_VID(pSma), tReadH.dFile.path, tstrerror(terrno));
-    return TSDB_CODE_FAILED;
-  }
-
-  char    smaKey[SMA_KEY_LEN] = {0};
-  void   *pSmaKey = &smaKey;
-  int64_t queryGroupId = 0;
-  tdEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey);
-
-  smaDebug("vgId:%d, get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma), tReadH.dFile.path,
-           *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN);
-
-  void   *result = NULL;
-  int32_t valueSize = 0;
-  if (!(result = smaGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize))) {
-    smaWarn("vgId:%d, get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s",
-            SMA_VID(pSma), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno));
-    smaCloseDBF(&tReadH.dFile);
-    return TSDB_CODE_FAILED;
-  }
-#endif
-
-#ifdef _TEST_SMA_PRINT_DEBUG_LOG_
-  for (uint32_t v = 0; v < valueSize; v += 8) {
-    smaWarn("vgId:%d, get sma data v[%d]=%" PRIi64, SMA_VID(pSma), v, *(int64_t *)POINTER_SHIFT(result, v));
-  }
-#endif
-  taosMemoryFreeClear(result);  // TODO: fill the result to output
-
-#if 0
-  int32_t nResult = 0;
-  int64_t lastKey = 0;
-
-  while (true) {
-    if (nResult >= nMaxResult) {
-      break;
-    }
-
-    // set and open the file according to the STSma param
-    if (tdSetAndOpenTSmaFile(&tReadH, queryWin)) {
-      char bTree[100] = "\0";
-      while (strncmp(bTree, "has more nodes", 100) == 0) {
-        if (nResult >= nMaxResult) {
-          break;
-        }
-        // tdGetDataFromBTree(bTree, queryWin, lastKey)
-        // fill the pData
-        ++nResult;
-      }
-    }
-  }
-#endif
-  // read data from file and fill the result
-  smaCloseDBF(&tReadH.dFile);
-  return TSDB_CODE_SUCCESS;
-}
-
-int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) {
-  SSmaCfg *pCfg = (SSmaCfg *)pMsg;
-
-  if (metaCreateTSma(SMA_META(pSma), version, pCfg) < 0) {
-    return -1;
-  }
-
-  tdTSmaAdd(pSma, 1);
-  return 0;
-}
-
-int32_t tdDropTSma(SSma *pSma, char *pMsg) {
-#if 0
-  SVDropTSmaReq vDropSmaReq = {0};
-  if (!tDeserializeSVDropTSmaReq(pMsg, &vDropSmaReq)) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  // TODO: send msg to stream computing to drop tSma
-  // if ((send msg to stream computing) < 0) {
-  //   tDestroyTSma(&vCreateSmaReq);
-  //   return -1;
-  // }
-  //
-
-  if (metaDropTSma(SMA_META(pSma), vDropSmaReq.indexUid) < 0) {
-    // TODO: handle error
-    return -1;
-  }
-
-  if (tdDropTSmaData(pSma, vDropSmaReq.indexUid) < 0) {
-    // TODO: handle error
-    return -1;
-  }
-
-  tdTSmaSub(pSma, 1);
-#endif
-
-  // TODO: return directly or go on follow steps?
-  return TSDB_CODE_SUCCESS;
-}
-
-static SSmaStatItem *tdNewSmaStatItem(int8_t state) {
-  SSmaStatItem *pItem = NULL;
-
-  pItem = (SSmaStatItem *)taosMemoryCalloc(1, sizeof(SSmaStatItem));
-  if (!pItem) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
-  }
-
-  pItem->state = state;
-  pItem->expiredWindows = taosHashInit(SMA_STATE_ITEM_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP),
-                                       true, HASH_ENTRY_LOCK);
-  if (!pItem->expiredWindows) {
-    taosMemoryFreeClear(pItem);
-    return NULL;
-  }
-
-  return pItem;
-}
-
-static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey,
-                                  int64_t version) {
-  SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid));
-  if (!pItem) {
-    // TODO: use TSDB_SMA_STAT_EXPIRED and update by stream computing later
-    pItem = tdNewSmaStatItem(TSDB_SMA_STAT_OK);  // TODO use the real state
-    if (!pItem) {
-      // Response to stream computing: OOM
-      // For query, if the indexUid not found, the TSDB should tell query module to query raw TS data.
-      return TSDB_CODE_FAILED;
-    }
-
-    // cache smaMeta
-    STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid);
-    if (!pTSma) {
-      terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META;
-      taosHashCleanup(pItem->expiredWindows);
-      taosMemoryFree(pItem);
-      smaWarn("vgId:%d, set expire window, get tsma meta failed for smaIndex %" PRIi64 " since %s", SMA_VID(pSma),
-              indexUid, tstrerror(terrno));
-      return TSDB_CODE_FAILED;
-    }
-    pItem->pTSma = pTSma;
-
-    if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) {
-      // If error occurs during put smaStatItem, free the resources of pItem
-      taosHashCleanup(pItem->expiredWindows);
-      taosMemoryFree(pItem);
-      return TSDB_CODE_FAILED;
-    }
-  } else if (!(pItem = *(SSmaStatItem **)pItem)) {
-    terrno = TSDB_CODE_INVALID_PTR;
-    return TSDB_CODE_FAILED;
-  }
-
-  if (taosHashPut(pItem->expiredWindows, &winSKey, sizeof(TSKEY), &version, sizeof(version)) != 0) {
-    // If error occurs during taosHashPut expired windows, remove the smaIndex from pSma->pSmaStat, thus TSDB would
-    // tell query module to query raw TS data.
-    // N.B.
-    //  1) It is assumed to be extemely little probability event of fail to taosHashPut.
-    //  2) This would solve the inconsistency to some extent, but not completely, unless we record all expired
-    // windows failed to put into hash table.
-    taosHashCleanup(pItem->expiredWindows);
-    taosMemoryFreeClear(pItem->pTSma);
-    taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid));
-    smaWarn("vgId:%d, smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window fail", SMA_VID(pSma), indexUid,
-            winSKey);
-    return TSDB_CODE_FAILED;
-  }
-
-  smaDebug("vgId:%d, smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window succeed", SMA_VID(pSma), indexUid,
-           winSKey);
-  return TSDB_CODE_SUCCESS;
-}
-
-/**
- * @brief Update expired window according to msg from stream computing module.
- *
- * @param pSma
- * @param msg SSubmitReq
- * @return int32_t
- */
-int32_t tdUpdateExpiredWindowImpl(SSma *pSma, const SSubmitReq *pMsg, int64_t version) {
-  // no time-range-sma, just return success
-  if (atomic_load_16(&SMA_TSMA_NUM(pSma)) <= 0) {
-    smaTrace("vgId:%d, not update expire window since no tSma", SMA_VID(pSma));
-    return TSDB_CODE_SUCCESS;
-  }
-
-  if (!SMA_META(pSma)) {
-    terrno = TSDB_CODE_INVALID_PTR;
-    smaError("vgId:%d, update expire window failed since no meta ptr", SMA_VID(pSma));
-    return TSDB_CODE_FAILED;
-  }
-
-  if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) < 0) {
-    smaError("vgId:%d, init sma env failed since %s", SMA_VID(pSma), terrstr(terrno));
-    terrno = TSDB_CODE_TDB_INIT_FAILED;
-    return TSDB_CODE_FAILED;
-  }
-
-  // Firstly, assume that tSma can only be created on super table/normal table.
-  // getActiveTimeWindow
-
-  SSmaEnv  *pEnv = SMA_TSMA_ENV(pSma);
-  SSmaStat *pStat = SMA_ENV_STAT(pEnv);
-  SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv);
-
-  TASSERT(pEnv && pStat && pItemsHash);
-
-  // basic procedure
-  // TODO: optimization
-  tdRefSmaStat(pSma, pStat);
-
-  SSubmitMsgIter msgIter = {0};
-  SSubmitBlk    *pBlock = NULL;
-  SInterval      interval = {0};
-  TSKEY          lastWinSKey = INT64_MIN;
-
-  if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) {
-    return TSDB_CODE_FAILED;
-  }
-
-  while (true) {
-    tGetSubmitMsgNext(&msgIter, &pBlock);
-    if (!pBlock) break;
-
-    STSmaWrapper *pSW = NULL;
-    STSma        *pTSma = NULL;
-
-    SSubmitBlkIter blkIter = {0};
-    if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) < 0) {
-      pSW = tFreeTSmaWrapper(pSW, false);
-      break;
-    }
-
-    while (true) {
-      STSRow *row = tGetSubmitBlkNext(&blkIter);
-      if (!row) {
-        pSW = tFreeTSmaWrapper(pSW, false);
-        break;
-      }
-      if (!pSW || (pTSma && (pTSma->tableUid != msgIter.suid))) {
-        if (pSW) {
-          pSW = tFreeTSmaWrapper(pSW, false);
-        }
-        if (!(pSW = metaGetSmaInfoByTable(SMA_META(pSma), msgIter.suid, false))) {
-          break;
-        }
-        if ((pSW->number) <= 0 || !pSW->tSma) {
-          pSW = tFreeTSmaWrapper(pSW, false);
-          break;
-        }
-
-        pTSma = pSW->tSma;
-
-        interval.interval = pTSma->interval;
-        interval.intervalUnit = pTSma->intervalUnit;
-        interval.offset = pTSma->offset;
-        interval.precision = SMA_TSDB_CFG(pSma)->precision;
-        interval.sliding = pTSma->sliding;
-        interval.slidingUnit = pTSma->slidingUnit;
-      }
-
-      // TODO: process multiple tsma for one table uid
-      TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision);
-
-      if (lastWinSKey != winSKey) {
-        lastWinSKey = winSKey;
-        if (tdSetExpiredWindow(pSma, pItemsHash, pTSma->indexUid, winSKey, version) < 0) {
-          pSW = tFreeTSmaWrapper(pSW, false);
-          tdUnRefSmaStat(pSma, pStat);
-          return TSDB_CODE_FAILED;
-        }
-      } else {
-        smaDebug("vgId:%d, smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window ignore as duplicated",
-                 SMA_VID(pSma), pTSma->indexUid, winSKey);
-      }
-    }
-  }
-
-  tdUnRefSmaStat(pSma, pStat);
-
-  return TSDB_CODE_SUCCESS;
-}

source/dnode/vnode/src/sma/smaTimeRange2.c

@@ -30,54 +30,8 @@ typedef STsdbCfg STSmaKeepCfg;
 
 #define SMA_STATE_ITEM_HASH_SLOT 32
 
-typedef struct {
-  SSma         *pSma;
-  SDBFile       dFile;
-  const SArray *pDataBlocks;  // sma data
-  int64_t       interval;     // interval with the precision of DB
-} STSmaWriteH;
-
-typedef struct {
-  int32_t iter;
-  int32_t fid;
-} SmaFsIter;
-
-typedef struct {
-  STsdb    *pTsdb;
-  SSma     *pSma;
-  SDBFile   dFile;
-  int64_t   interval;   // interval with the precision of DB
-  int32_t   blockSize;  // size of SMA block item
-  int32_t   days;
-  int8_t    storageLevel;
-  SmaFsIter smaFsIter;
-} STSmaReadH;
-
-typedef enum {
-  SMA_STORAGE_LEVEL_TSDB = 0,     // use days of self-defined  e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f200.tsma
-  SMA_STORAGE_LEVEL_DFILESET = 1  // use days of TS data       e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma
-} ESmaStorageLevel;
-
 // static func
 
-static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted);
-static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval);
-static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval,
-                                int8_t intervalUnit);
-static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit);
-static void    tdDestroyTSmaWriteH(STSmaWriteH *pSmaH);
-static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel);
-static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid);
-static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey);
-static bool    tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey);
-static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen,
-                                  TXN *txn);
-// expired window
-
-static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, int64_t version);
-static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey);
-static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid);
-
 /**
  * @brief Judge the tsma file split days
  *
@@ -87,7 +41,7 @@ static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid);
  * @param days unit is minute
  * @return int32_t
  */
-int32_t tdGetTSmaDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) {
+int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) {
   SDecoder coder = {0};
   tDecoderInit(&coder, pCont, contLen);
 
@@ -130,225 +84,6 @@ _err:
 
 // implementation
 
-/**
- * @brief
- *
- * @param pSmaH
- * @param pSma
- * @param interval
- * @param intervalUnit
- * @return int32_t
- */
-static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit) {
-  STSmaKeepCfg *pCfg = SMA_TSDB_CFG(pSma);
-  pSmaH->pSma = pSma;
-  pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true);
-  pSmaH->storageLevel = tdGetSmaStorageLevel(pCfg, interval);
-  pSmaH->days = tdGetTSmaDays(pSma, pSmaH->interval, pSmaH->storageLevel);
-  return TSDB_CODE_SUCCESS;
-}
-
-/**
- * @brief Init of tSma FS
- *
- * @param pReadH
- * @param indexUid
- * @param skey
- * @return int32_t
- */
-static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey) {
-  SSma *pSma = pSmaH->pSma;
-
-  int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, SMA_TSDB_CFG(pSma)->precision));
-  char    tSmaFile[TSDB_FILENAME_LEN] = {0};
-  snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, SMA_VID(pSma), fid);
-  pSmaH->dFile.path = strdup(tSmaFile);
-  pSmaH->smaFsIter.iter = 0;
-  pSmaH->smaFsIter.fid = fid;
-  return TSDB_CODE_SUCCESS;
-}
-
-/**
- * @brief Set and open tSma file if it has key locates in queryWin.
- *
- * @param pReadH
- * @param param
- * @param queryWin
- * @return true
- * @return false
- */
-static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) {
-  // SArray *smaFs = pReadH->pTsdb->fs->cstatus->sf;
-  // int32_t nSmaFs = taosArrayGetSize(smaFs);
-
-  smaCloseDBF(&pReadH->dFile);
-
-#if 0
-  while (pReadH->smaFsIter.iter < nSmaFs) {
-    void *pSmaFile = taosArrayGet(smaFs, pReadH->smaFsIter.iter);
-    if (pSmaFile) {  // match(indexName, queryWindow)
-      // TODO: select the file by index_name ...
-      pReadH->dFile = pSmaFile;
-      ++pReadH->smaFsIter.iter;
-      break;
-    }
-    ++pReadH->smaFsIter.iter;
-  }
-
-  if (pReadH->pDFile) {
-    tdDebug("vg%d: smaFile %s matched", REPO_ID(pReadH->pTsdb), "[pSmaFile dir]");
-    return true;
-  }
-#endif
-
-  return false;
-}
-
-/**
- * @brief Approximate value for week/month/year.
- *
- * @param interval
- * @param intervalUnit
- * @param precision
- * @param adjusted Interval already adjusted according to DB precision
- * @return int64_t
- */
-static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted) {
-  if (adjusted) {
-    return interval;
-  }
-
-  switch (intervalUnit) {
-    case TIME_UNIT_YEAR:  // approximate value
-      interval *= 365 * 86400 * 1e3;
-      break;
-    case TIME_UNIT_MONTH:  // approximate value
-      interval *= 30 * 86400 * 1e3;
-      break;
-    case TIME_UNIT_WEEK:  // approximate value
-      interval *= 7 * 86400 * 1e3;
-      break;
-    case TIME_UNIT_DAY:  // the interval for tSma calculation must <= day
-      interval *= 86400 * 1e3;
-      break;
-    case TIME_UNIT_HOUR:
-      interval *= 3600 * 1e3;
-      break;
-    case TIME_UNIT_MINUTE:
-      interval *= 60 * 1e3;
-      break;
-    case TIME_UNIT_SECOND:
-      interval *= 1e3;
-      break;
-    default:
-      break;
-  }
-
-  switch (precision) {
-    case TSDB_TIME_PRECISION_MILLI:
-      if (TIME_UNIT_MICROSECOND == intervalUnit) {  // us
-        return interval / 1e3;
-      } else if (TIME_UNIT_NANOSECOND == intervalUnit) {  //  nano second
-        return interval / 1e6;
-      } else {  // ms
-        return interval;
-      }
-      break;
-    case TSDB_TIME_PRECISION_MICRO:
-      if (TIME_UNIT_MICROSECOND == intervalUnit) {  // us
-        return interval;
-      } else if (TIME_UNIT_NANOSECOND == intervalUnit) {  //  ns
-        return interval / 1e3;
-      } else {  // ms
-        return interval * 1e3;
-      }
-      break;
-    case TSDB_TIME_PRECISION_NANO:
-      if (TIME_UNIT_MICROSECOND == intervalUnit) {  // us
-        return interval * 1e3;
-      } else if (TIME_UNIT_NANOSECOND == intervalUnit) {  // ns
-        return interval;
-      } else {  // ms
-        return interval * 1e6;
-      }
-      break;
-    default:                                        // ms
-      if (TIME_UNIT_MICROSECOND == intervalUnit) {  // us
-        return interval / 1e3;
-      } else if (TIME_UNIT_NANOSECOND == intervalUnit) {  //  ns
-        return interval / 1e6;
-      } else {  // ms
-        return interval;
-      }
-      break;
-  }
-  return interval;
-}
-
-static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval,
-                                int8_t intervalUnit) {
-  pSmaH->pSma = pSma;
-  pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true);
-  pSmaH->pDataBlocks = pDataBlocks;
-  pSmaH->dFile.fid = SMA_IVLD_FID;
-  return TSDB_CODE_SUCCESS;
-}
-
-static void tdDestroyTSmaWriteH(STSmaWriteH *pSmaH) {
-  if (pSmaH) {
-    smaCloseDBF(&pSmaH->dFile);
-  }
-}
-
-static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid) {
-  SSma *pSma = pSmaH->pSma;
-  ASSERT(!pSmaH->dFile.path && !pSmaH->dFile.pDB);
-
-  pSmaH->dFile.fid = fid;
-  char tSmaFile[TSDB_FILENAME_LEN] = {0};
-  snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, SMA_VID(pSma), fid);
-  pSmaH->dFile.path = strdup(tSmaFile);
-
-  return TSDB_CODE_SUCCESS;
-}
-
-/**
- * @brief
- *
- * @param pSma
- * @param interval Interval calculated by DB's precision
- * @param storageLevel
- * @return int32_t
- */
-static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel) {
-  STsdbCfg *pCfg = SMA_TSDB_CFG(pSma);
-  int32_t   daysPerFile = pCfg->days;  // unit is minute
-
-  if (storageLevel == SMA_STORAGE_LEVEL_TSDB) {
-    int32_t minutes = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerMin[pCfg->precision]);
-    if (minutes > SMA_STORAGE_TSDB_MINUTES) {
-      daysPerFile = SMA_STORAGE_TSDB_MINUTES;
-    }
-  }
-
-  return daysPerFile;
-}
-
-/**
- * @brief Judge the tSma storage level
- *
- * @param pCfg
- * @param interval
- * @return int32_t
- */
-static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval) {
-  int64_t mInterval = convertTimeFromPrecisionToUnit(interval, pCfg->precision, TIME_UNIT_MINUTE);
-  if (pCfg->days / mInterval >= SMA_STORAGE_SPLIT_FACTOR) {
-    return SMA_STORAGE_LEVEL_DFILESET;
-  }
-  return SMA_STORAGE_LEVEL_TSDB;
-}
-
 /**
  * @brief Insert/Update Time-range-wise SMA data.
  *  - If interval < SMA_STORAGE_SPLIT_HOURS(e.g. 24), save the SMA data as a part of DFileSet to e.g.
@@ -362,28 +97,28 @@ static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval) {
  * @return int32_t
  */
 int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
-  STsdbCfg     *pCfg = SMA_TSDB_CFG(pSma);
+  STsdbCfg *pCfg = SMA_TSDB_CFG(pSma);
+
   const SArray *pDataBlocks = (const SArray *)msg;
-  int64_t       testSkey = TSKEY_INITIAL_VAL;
 
   // TODO: destroy SSDataBlocks(msg)
 
   // For super table aggregation, the sma data is stored in vgroup calculated from the hash value of stable name. Thus
   // the sma data would arrive ahead of the update-expired-window msg.
-  if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) != TSDB_CODE_SUCCESS) {
+  if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE, false) != TSDB_CODE_SUCCESS) {
     terrno = TSDB_CODE_TDB_INIT_FAILED;
     return TSDB_CODE_FAILED;
   }
 
   if (!pDataBlocks) {
     terrno = TSDB_CODE_INVALID_PTR;
-    smaWarn("vgId:%d insert tSma data failed since pDataBlocks is NULL", SMA_VID(pSma));
+    smaWarn("vgId:%d, insert tsma data failed since pDataBlocks is NULL", SMA_VID(pSma));
     return terrno;
   }
 
   if (taosArrayGetSize(pDataBlocks) <= 0) {
     terrno = TSDB_CODE_INVALID_PARA;
-    smaWarn("vgId:%d insert tSma data failed since pDataBlocks is empty", SMA_VID(pSma));
+    smaWarn("vgId:%d, insert tsma data failed since pDataBlocks is empty", SMA_VID(pSma));
     return TSDB_CODE_FAILED;
   }
 
@@ -398,463 +133,15 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) {
   }
 
   if (!pItem || !(pItem = *(SSmaStatItem **)pItem) || tdSmaStatIsDropped(pItem)) {
-    terrno = TSDB_CODE_TDB_INVALID_SMA_STAT;
+    terrno = TSDB_CODE_TSMA_INVALID_STAT;
     tdUnRefSmaStat(pSma, pStat);
     return TSDB_CODE_FAILED;
   }
 
-  STSma      *pTSma = pItem->pTSma;
-  STSmaWriteH tSmaH = {0};
-
-  if (tdInitTSmaWriteH(&tSmaH, pSma, pDataBlocks, pTSma->interval, pTSma->intervalUnit) != 0) {
-    return TSDB_CODE_FAILED;
-  }
-
-  char rPath[TSDB_FILENAME_LEN] = {0};
-  char aPath[TSDB_FILENAME_LEN] = {0};
-  snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid);
-  tfsAbsoluteName(SMA_TFS(pSma), SMA_ENV_DID(pEnv), rPath, aPath);
-  if (!taosCheckExistFile(aPath)) {
-    if (tfsMkdirRecurAt(SMA_TFS(pSma), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) {
-      tdUnRefSmaStat(pSma, pStat);
-      return TSDB_CODE_FAILED;
-    }
-  }
-
-  // Step 1: Judge the storage level and days
-  int32_t storageLevel = tdGetSmaStorageLevel(pCfg, tSmaH.interval);
-  int32_t minutePerFile = tdGetTSmaDays(pSma, tSmaH.interval, storageLevel);
-
-  char    smaKey[SMA_KEY_LEN] = {0};  // key: skey + groupId
-  char    dataBuf[512] = {0};         // val: aggr data // TODO: handle 512 buffer?
-  void   *pDataBuf = NULL;
-  int32_t sz = taosArrayGetSize(pDataBlocks);
-  for (int32_t i = 0; i < sz; ++i) {
-    SSDataBlock *pDataBlock = taosArrayGet(pDataBlocks, i);
-    int32_t      colNum = pDataBlock->info.numOfCols;
-    int32_t      rows = pDataBlock->info.rows;
-    int32_t      rowSize = pDataBlock->info.rowSize;
-    int64_t      groupId = pDataBlock->info.groupId;
-    for (int32_t j = 0; j < rows; ++j) {
-      printf("|");
-      TSKEY skey = TSKEY_INITIAL_VAL;  //  the start key of TS window by interval
-      void *pSmaKey = &smaKey;
-      bool  isStartKey = false;
-
-      int32_t tlen = 0;     // reset the len
-      pDataBuf = &dataBuf;  // reset the buf
-      for (int32_t k = 0; k < colNum; ++k) {
-        SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
-        void            *var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
-        switch (pColInfoData->info.type) {
-          case TSDB_DATA_TYPE_TIMESTAMP:
-            if (!isStartKey) {
-              isStartKey = true;
-              skey = *(TSKEY *)var;
-              testSkey = skey;
-              printf("= skey %" PRIi64 " groupId = %" PRIi64 "|", skey, groupId);
-              tdEncodeTSmaKey(groupId, skey, &pSmaKey);
-            } else {
-              printf(" %" PRIi64 " |", *(int64_t *)var);
-              tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var);
-              break;
-            }
-            break;
-          case TSDB_DATA_TYPE_BOOL:
-          case TSDB_DATA_TYPE_UTINYINT:
-            printf(" %15d |", *(uint8_t *)var);
-            tlen += taosEncodeFixedU8(&pDataBuf, *(uint8_t *)var);
-            break;
-          case TSDB_DATA_TYPE_TINYINT:
-            printf(" %15d |", *(int8_t *)var);
-            tlen += taosEncodeFixedI8(&pDataBuf, *(int8_t *)var);
-            break;
-          case TSDB_DATA_TYPE_SMALLINT:
-            printf(" %15d |", *(int16_t *)var);
-            tlen += taosEncodeFixedI16(&pDataBuf, *(int16_t *)var);
-            break;
-          case TSDB_DATA_TYPE_USMALLINT:
-            printf(" %15d |", *(uint16_t *)var);
-            tlen += taosEncodeFixedU16(&pDataBuf, *(uint16_t *)var);
-            break;
-          case TSDB_DATA_TYPE_INT:
-            printf(" %15d |", *(int32_t *)var);
-            tlen += taosEncodeFixedI32(&pDataBuf, *(int32_t *)var);
-            break;
-          case TSDB_DATA_TYPE_FLOAT:
-            printf(" %15f |", *(float *)var);
-            tlen += taosEncodeBinary(&pDataBuf, var, sizeof(float));
-            break;
-          case TSDB_DATA_TYPE_UINT:
-            printf(" %15u |", *(uint32_t *)var);
-            tlen += taosEncodeFixedU32(&pDataBuf, *(uint32_t *)var);
-            break;
-          case TSDB_DATA_TYPE_BIGINT:
-            printf(" %15ld |", *(int64_t *)var);
-            tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var);
-            break;
-          case TSDB_DATA_TYPE_DOUBLE:
-            printf(" %15lf |", *(double *)var);
-            tlen += taosEncodeBinary(&pDataBuf, var, sizeof(double));
-          case TSDB_DATA_TYPE_UBIGINT:
-            printf(" %15lu |", *(uint64_t *)var);
-            tlen += taosEncodeFixedU64(&pDataBuf, *(uint64_t *)var);
-            break;
-          case TSDB_DATA_TYPE_NCHAR: {
-            char tmpChar[100] = {0};
-            strncpy(tmpChar, varDataVal(var), varDataLen(var));
-            printf(" %s |", tmpChar);
-            tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var));
-            break;
-          }
-          case TSDB_DATA_TYPE_VARCHAR: {  // TSDB_DATA_TYPE_BINARY
-            char tmpChar[100] = {0};
-            strncpy(tmpChar, varDataVal(var), varDataLen(var));
-            printf(" %s |", tmpChar);
-            tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var));
-            break;
-          }
-          case TSDB_DATA_TYPE_VARBINARY:
-            // TODO: add binary/varbinary
-            TASSERT(0);
-          default:
-            printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type);
-            TASSERT(0);
-            break;
-        }
-      }
-      printf("\n");
-      // if ((tlen > 0) && (skey != TSKEY_INITIAL_VAL)) {
-      if (tlen > 0) {
-        int32_t fid = (int32_t)(TSDB_KEY_FID(skey, minutePerFile, pCfg->precision));
-
-        // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index
-        // file
-        //         - Set and open the DFile or the B+Tree file
-        // TODO: tsdbStartTSmaCommit();
-        if (fid != tSmaH.dFile.fid) {
-          if (tSmaH.dFile.fid != SMA_IVLD_FID) {
-            tdSmaEndCommit(pEnv);
-            smaCloseDBF(&tSmaH.dFile);
-          }
-          tdSetTSmaDataFile(&tSmaH, indexUid, fid);
-          smaDebug("@@@ vgId:%d write to DBF %s, days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi32
-                   " queryKey:%" PRIi64,
-                   SMA_VID(pSma), tSmaH.dFile.path, minutePerFile, tSmaH.interval, storageLevel, testSkey);
-          if (smaOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) {
-            smaWarn("vgId:%d open DB file %s failed since %s", SMA_VID(pSma),
-                    tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno));
-            tdDestroyTSmaWriteH(&tSmaH);
-            tdUnRefSmaStat(pSma, pStat);
-            return TSDB_CODE_FAILED;
-          }
-          tdSmaBeginCommit(pEnv);
-        }
-
-        if (tdInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen, &pEnv->txn) != 0) {
-          smaWarn("vgId:%d insert tsma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64
-                  " since %s",
-                  SMA_VID(pSma), indexUid, skey, groupId, tstrerror(terrno));
-          tdSmaEndCommit(pEnv);
-          tdDestroyTSmaWriteH(&tSmaH);
-          tdUnRefSmaStat(pSma, pStat);
-          return TSDB_CODE_FAILED;
-        }
-
-        smaDebug("vgId:%d insert tsma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64,
-                 SMA_VID(pSma), indexUid, skey, groupId);
-        // TODO:tsdbEndTSmaCommit();
-
-        // Step 3: reset the SSmaStat
-        tdResetExpiredWindow(pSma, pStat, indexUid, skey);
-      } else {
-        smaWarn("vgId:%d invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64,
-                SMA_VID(pSma), skey, tlen, indexUid);
-      }
-    }
-  }
-  tdSmaEndCommit(pEnv);  // TODO: not commit for every insert
-  tdDestroyTSmaWriteH(&tSmaH);
-  tdUnRefSmaStat(pSma, pStat);
-
-  return TSDB_CODE_SUCCESS;
-}
-
-int32_t tdDropTSmaData(SSma *pSma, int64_t indexUid) {
-  int32_t code = TSDB_CODE_SUCCESS;
-  if ((code = tdDropTSmaDataImpl(pSma, indexUid)) < 0) {
-    smaWarn("vgId:%d drop tSma data failed since %s", SMA_VID(pSma), tstrerror(terrno));
-  }
-  return code;
-}
-
-/**
- * @brief Insert TSma data blocks to DB File build by B+Tree
- *
- * @param pSmaH
- * @param smaKey  tableUid-colId-skeyOfWindow(8-2-8)
- * @param keyLen
- * @param pData
- * @param dataLen
- * @return int32_t
- */
-static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen,
-                                  TXN *txn) {
-  SDBFile *pDBFile = &pSmaH->dFile;
-
-  // TODO: insert tsma data blocks into B+Tree(TTB)
-  if (smaSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) {
-    smaWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail",
-            SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
-    return TSDB_CODE_FAILED;
-  }
-  smaDebug("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed",
-           SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen);
-
-#ifdef _TEST_SMA_PRINT_DEBUG_LOG_
-  uint32_t valueSize = 0;
-  void    *data = tdGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize);
-  ASSERT(data != NULL);
-  for (uint32_t v = 0; v < valueSize; v += 8) {
-    smaWarn("vgId:%d insert sma data val[%d] %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v));
-  }
-#endif
-  return TSDB_CODE_SUCCESS;
-}
-
-/**
- * @brief When sma data received from stream computing, make the relative expired window valid.
- *
- * @param pSma
- * @param pStat
- * @param indexUid
- * @param skey
- * @return int32_t
- */
-static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey) {
-  SSmaStatItem *pItem = NULL;
-
-  tdRefSmaStat(pSma, pStat);
-
-  if (pStat && SMA_STAT_ITEMS(pStat)) {
-    pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid));
-  }
-  if ((pItem) && ((pItem = *(SSmaStatItem **)pItem))) {
-    // pItem resides in hash buffer all the time unless drop sma index
-    // TODO: multithread protect
-    if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) {
-      // error handling
-      tdUnRefSmaStat(pSma, pStat);
-      smaWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma), skey,
-              indexUid);
-      return TSDB_CODE_FAILED;
-    }
-    smaDebug("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", SMA_VID(pSma),
-             skey, indexUid);
-    // TODO: use a standalone interface to received state upate notification from stream computing module.
-    /**
-     * @brief state
-     *  - When SMA env init in TSDB, its status is TSDB_SMA_STAT_OK.
-     *  - In startup phase of stream computing module, it should notify the SMA env in TSDB to expired if needed(e.g.
-     * when batch data caculation not finised)
-     *  - When TSDB_SMA_STAT_OK, the stream computing module should also notify that to the SMA env in TSDB.
-     */
-    pItem->state = TSDB_SMA_STAT_OK;
-  } else {
-    // error handling
-    tdUnRefSmaStat(pSma, pStat);
-    smaWarn("vgId:%d expired window %" PRIi64 " not exists for sma index %" PRIi64, SMA_VID(pSma), skey, indexUid);
-    return TSDB_CODE_FAILED;
-  }
-
-  tdUnRefSmaStat(pSma, pStat);
-  return TSDB_CODE_SUCCESS;
-}
-
-/**
- * @brief Drop tSma data and local cache
- *        - insert/query reference
- * @param pSma
- * @param msg
- * @return int32_t
- */
-static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid) {
-  SSmaEnv *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma));
-
-  // clear local cache
-  if (pEnv) {
-    smaDebug("vgId:%d drop tSma local cache for %" PRIi64, SMA_VID(pSma), indexUid);
-
-    SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid));
-    if ((pItem) || ((pItem = *(SSmaStatItem **)pItem))) {
-      if (tdSmaStatIsDropped(pItem)) {
-        smaDebug("vgId:%d tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid);
-        return TSDB_CODE_TDB_INVALID_ACTION;  // TODO: duplicate drop msg would be intercepted by mnode
-      }
-
-      tdWLockSmaEnv(pEnv);
-      if (tdSmaStatIsDropped(pItem)) {
-        tdUnLockSmaEnv(pEnv);
-        smaDebug("vgId:%d tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid);
-        return TSDB_CODE_TDB_INVALID_ACTION;  // TODO: duplicate drop msg would be intercepted by mnode
-      }
-      tdSmaStatSetDropped(pItem);
-      tdUnLockSmaEnv(pEnv);
-
-      int32_t nSleep = 0;
-      int32_t refVal = INT32_MAX;
-      while (true) {
-        if ((refVal = T_REF_VAL_GET(SMA_ENV_STAT(pEnv))) <= 0) {
-          smaDebug("vgId:%d drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal);
-          break;
-        }
-        smaDebug("vgId:%d wait 1s to drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal);
-        taosSsleep(1);
-        if (++nSleep > SMA_DROP_EXPIRED_TIME) {
-          smaDebug("vgId:%d drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep, refVal);
-          break;
-        };
-      }
-
-      tdFreeSmaStatItem(pItem);
-      smaDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64 " in local cache", SMA_VID(pSma), indexUid);
-    }
-  }
-  // clear sma data files
-  // TODO:
-  return TSDB_CODE_SUCCESS;
-}
-
-/**
- * @brief
- *
- * @param pSma Return the data between queryWin and fill the pData.
- * @param pData
- * @param indexUid
- * @param pQuerySKey
- * @param nMaxResult The query invoker should control the nMaxResult need to return to avoid OOM.
- * @return int32_t
- */
-int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) {
-  SSmaEnv  *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma));
-  SSmaStat *pStat = NULL;
-
-  if (!pEnv) {
-    terrno = TSDB_CODE_INVALID_PTR;
-    smaWarn("vgId:%d getTSmaDataImpl failed since pTSmaEnv is NULL", SMA_VID(pSma));
-    return TSDB_CODE_FAILED;
-  }
-
-  pStat = SMA_ENV_STAT(pEnv);
-
-  tdRefSmaStat(pSma, pStat);
-  SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid));
-  if (!pItem || !(pItem = *(SSmaStatItem **)pItem)) {
-    // Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if
-    // it's NULL.
-    tdUnRefSmaStat(pSma, pStat);
-    terrno = TSDB_CODE_TDB_INVALID_ACTION;
-    smaDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64, SMA_VID(pSma), indexUid);
-    return TSDB_CODE_FAILED;
-  }
-
-#if 0
-  int32_t nQueryWin = taosArrayGetSize(pQuerySKey);
-  for (int32_t n = 0; n < nQueryWin; ++n) {
-    TSKEY skey = taosArrayGet(pQuerySKey, n);
-    if (taosHashGet(pItem->expiredWindows, &skey, sizeof(TSKEY))) {
-      // TODO: mark this window as expired.
-    }
-  }
-#endif
-
-#if 1
-  int8_t smaStat = 0;
-  if (!tdSmaStatIsOK(pItem, &smaStat)) {  // TODO: multiple check for large scale sma query
-    tdUnRefSmaStat(pSma, pStat);
-    terrno = TSDB_CODE_TDB_INVALID_SMA_STAT;
-    smaWarn("vgId:%d getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, SMA_VID(pSma), indexUid,
-            tstrerror(terrno), smaStat);
-    return TSDB_CODE_FAILED;
-  }
-
-  if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY))) {
-    // TODO: mark this window as expired.
-    smaDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma), querySKey,
-             indexUid);
-  } else {
-    smaDebug("vgId:%d skey %" PRIi64 " of window not in expired window for index %" PRIi64, SMA_VID(pSma), querySKey,
-             indexUid);
-  }
-
   STSma *pTSma = pItem->pTSma;
-#endif
-
-#if 1
-  STSmaReadH tReadH = {0};
-  tdInitTSmaReadH(&tReadH, pSma, pTSma->interval, pTSma->intervalUnit);
-  smaCloseDBF(&tReadH.dFile);
 
   tdUnRefSmaStat(pSma, pStat);
 
-  tdInitTSmaFile(&tReadH, indexUid, querySKey);
-  smaDebug("### vgId:%d read from DBF %s  days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi8 " queryKey:%" PRIi64,
-           SMA_VID(pSma), tReadH.dFile.path, tReadH.days, tReadH.interval, tReadH.storageLevel, querySKey);
-  if (smaOpenDBF(pEnv->dbEnv, &tReadH.dFile) != 0) {
-    smaWarn("vgId:%d open DBF %s failed since %s", SMA_VID(pSma), tReadH.dFile.path, tstrerror(terrno));
-    return TSDB_CODE_FAILED;
-  }
-
-  char    smaKey[SMA_KEY_LEN] = {0};
-  void   *pSmaKey = &smaKey;
-  int64_t queryGroupId = 0;
-  tdEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey);
-
-  smaDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma), tReadH.dFile.path,
-           *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN);
-
-  void   *result = NULL;
-  int32_t valueSize = 0;
-  if (!(result = smaGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize))) {
-    smaWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s",
-            SMA_VID(pSma), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno));
-    smaCloseDBF(&tReadH.dFile);
-    return TSDB_CODE_FAILED;
-  }
-#endif
-
-#ifdef _TEST_SMA_PRINT_DEBUG_LOG_
-  for (uint32_t v = 0; v < valueSize; v += 8) {
-    smaWarn("vgId:%d get sma data v[%d]=%" PRIi64, SMA_VID(pSma), v, *(int64_t *)POINTER_SHIFT(result, v));
-  }
-#endif
-  taosMemoryFreeClear(result);  // TODO: fill the result to output
-
-#if 0
-  int32_t nResult = 0;
-  int64_t lastKey = 0;
-
-  while (true) {
-    if (nResult >= nMaxResult) {
-      break;
-    }
-
-    // set and open the file according to the STSma param
-    if (tdSetAndOpenTSmaFile(&tReadH, queryWin)) {
-      char bTree[100] = "\0";
-      while (strncmp(bTree, "has more nodes", 100) == 0) {
-        if (nResult >= nMaxResult) {
-          break;
-        }
-        // tdGetDataFromBTree(bTree, queryWin, lastKey)
-        // fill the pData
-        ++nResult;
-      }
-    }
-  }
-#endif
-  // read data from file and fill the result
-  smaCloseDBF(&tReadH.dFile);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -865,224 +152,19 @@ int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) {
     return -1;
   }
 
-  tdTSmaAdd(pSma, 1);
-  return 0;
-}
-
-int32_t tdDropTSma(SSma *pSma, char *pMsg) {
-#if 0
-  SVDropTSmaReq vDropSmaReq = {0};
-  if (!tDeserializeSVDropTSmaReq(pMsg, &vDropSmaReq)) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  // TODO: send msg to stream computing to drop tSma
-  // if ((send msg to stream computing) < 0) {
-  //   tDestroyTSma(&vCreateSmaReq);
-  //   return -1;
-  // }
-  //
+  if (TD_VID(pSma->pVnode) == pCfg->dstVgId) {
+    // create stable to save tsma result in dstVgId
+    SVCreateStbReq pReq = {0};
+    pReq.name = pCfg->dstTbName;
+    pReq.suid = pCfg->dstTbUid;
+    pReq.schemaRow = pCfg->schemaRow;
+    pReq.schemaTag = pCfg->schemaTag;
 
-  if (metaDropTSma(SMA_META(pSma), vDropSmaReq.indexUid) < 0) {
-    // TODO: handle error
-    return -1;
-  }
-
-  if (tdDropTSmaData(pSma, vDropSmaReq.indexUid) < 0) {
-    // TODO: handle error
-    return -1;
-  }
-
-  tdTSmaSub(pSma, 1);
-#endif
-
-  // TODO: return directly or go on follow steps?
-  return TSDB_CODE_SUCCESS;
-}
-
-static SSmaStatItem *tdNewSmaStatItem(int8_t state) {
-  SSmaStatItem *pItem = NULL;
-
-  pItem = (SSmaStatItem *)taosMemoryCalloc(1, sizeof(SSmaStatItem));
-  if (!pItem) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
-  }
-
-  pItem->state = state;
-  pItem->expiredWindows = taosHashInit(SMA_STATE_ITEM_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP),
-                                       true, HASH_ENTRY_LOCK);
-  if (!pItem->expiredWindows) {
-    taosMemoryFreeClear(pItem);
-    return NULL;
-  }
-
-  return pItem;
-}
-
-static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey,
-                                  int64_t version) {
-  SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid));
-  if (!pItem) {
-    // TODO: use TSDB_SMA_STAT_EXPIRED and update by stream computing later
-    pItem = tdNewSmaStatItem(TSDB_SMA_STAT_OK);  // TODO use the real state
-    if (!pItem) {
-      // Response to stream computing: OOM
-      // For query, if the indexUid not found, the TSDB should tell query module to query raw TS data.
-      return TSDB_CODE_FAILED;
-    }
-
-    // cache smaMeta
-    STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid);
-    if (!pTSma) {
-      terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META;
-      taosHashCleanup(pItem->expiredWindows);
-      taosMemoryFree(pItem);
-      smaWarn("vgId:%d set expire window, get tsma meta failed for smaIndex %" PRIi64 " since %s", SMA_VID(pSma),
-              indexUid, tstrerror(terrno));
-      return TSDB_CODE_FAILED;
-    }
-    pItem->pTSma = pTSma;
-
-    if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) {
-      // If error occurs during put smaStatItem, free the resources of pItem
-      taosHashCleanup(pItem->expiredWindows);
-      taosMemoryFree(pItem);
-      return TSDB_CODE_FAILED;
-    }
-  } else if (!(pItem = *(SSmaStatItem **)pItem)) {
-    terrno = TSDB_CODE_INVALID_PTR;
-    return TSDB_CODE_FAILED;
-  }
-
-  if (taosHashPut(pItem->expiredWindows, &winSKey, sizeof(TSKEY), &version, sizeof(version)) != 0) {
-    // If error occurs during taosHashPut expired windows, remove the smaIndex from pSma->pSmaStat, thus TSDB would
-    // tell query module to query raw TS data.
-    // N.B.
-    //  1) It is assumed to be extemely little probability event of fail to taosHashPut.
-    //  2) This would solve the inconsistency to some extent, but not completely, unless we record all expired
-    // windows failed to put into hash table.
-    taosHashCleanup(pItem->expiredWindows);
-    taosMemoryFreeClear(pItem->pTSma);
-    taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid));
-    smaWarn("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window fail", SMA_VID(pSma), indexUid,
-            winSKey);
-    return TSDB_CODE_FAILED;
-  }
-
-  smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window succeed", SMA_VID(pSma), indexUid,
-           winSKey);
-  return TSDB_CODE_SUCCESS;
-}
-
-/**
- * @brief Update expired window according to msg from stream computing module.
- *
- * @param pSma
- * @param msg SSubmitReq
- * @return int32_t
- */
-int32_t tdUpdateExpiredWindowImpl(SSma *pSma, const SSubmitReq *pMsg, int64_t version) {
-  // no time-range-sma, just return success
-  if (atomic_load_16(&SMA_TSMA_NUM(pSma)) <= 0) {
-    smaTrace("vgId:%d not update expire window since no tSma", SMA_VID(pSma));
-    return TSDB_CODE_SUCCESS;
-  }
-
-  if (!SMA_META(pSma)) {
-    terrno = TSDB_CODE_INVALID_PTR;
-    smaError("vgId:%d update expire window failed since no meta ptr", SMA_VID(pSma));
-    return TSDB_CODE_FAILED;
-  }
-
-  if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) < 0) {
-    smaError("vgId:%d init sma env failed since %s", SMA_VID(pSma), terrstr(terrno));
-    terrno = TSDB_CODE_TDB_INIT_FAILED;
-    return TSDB_CODE_FAILED;
-  }
-
-  // Firstly, assume that tSma can only be created on super table/normal table.
-  // getActiveTimeWindow
-
-  SSmaEnv  *pEnv = SMA_TSMA_ENV(pSma);
-  SSmaStat *pStat = SMA_ENV_STAT(pEnv);
-  SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv);
-
-  TASSERT(pEnv && pStat && pItemsHash);
-
-  // basic procedure
-  // TODO: optimization
-  tdRefSmaStat(pSma, pStat);
-
-  SSubmitMsgIter msgIter = {0};
-  SSubmitBlk    *pBlock = NULL;
-  SInterval      interval = {0};
-  TSKEY          lastWinSKey = INT64_MIN;
-
-  if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) {
-    return TSDB_CODE_FAILED;
-  }
-
-  while (true) {
-    tGetSubmitMsgNext(&msgIter, &pBlock);
-    if (!pBlock) break;
-
-    STSmaWrapper *pSW = NULL;
-    STSma        *pTSma = NULL;
-
-    SSubmitBlkIter blkIter = {0};
-    if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) < 0) {
-      pSW = tFreeTSmaWrapper(pSW, false);
-      break;
-    }
-
-    while (true) {
-      STSRow *row = tGetSubmitBlkNext(&blkIter);
-      if (!row) {
-        pSW = tFreeTSmaWrapper(pSW, false);
-        break;
-      }
-      if (!pSW || (pTSma && (pTSma->tableUid != msgIter.suid))) {
-        if (pSW) {
-          pSW = tFreeTSmaWrapper(pSW, false);
-        }
-        if (!(pSW = metaGetSmaInfoByTable(SMA_META(pSma), msgIter.suid, false))) {
-          break;
-        }
-        if ((pSW->number) <= 0 || !pSW->tSma) {
-          pSW = tFreeTSmaWrapper(pSW, false);
-          break;
-        }
-
-        pTSma = pSW->tSma;
-
-        interval.interval = pTSma->interval;
-        interval.intervalUnit = pTSma->intervalUnit;
-        interval.offset = pTSma->offset;
-        interval.precision = SMA_TSDB_CFG(pSma)->precision;
-        interval.sliding = pTSma->sliding;
-        interval.slidingUnit = pTSma->slidingUnit;
-      }
-
-      // TODO: process multiple tsma for one table uid
-      TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision);
-
-      if (lastWinSKey != winSKey) {
-        lastWinSKey = winSKey;
-        if (tdSetExpiredWindow(pSma, pItemsHash, pTSma->indexUid, winSKey, version) < 0) {
-          pSW = tFreeTSmaWrapper(pSW, false);
-          tdUnRefSmaStat(pSma, pStat);
-          return TSDB_CODE_FAILED;
-        }
-      } else {
-        smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window ignore as duplicated",
-                 SMA_VID(pSma), pTSma->indexUid, winSKey);
-      }
+    if (metaCreateSTable(SMA_META(pSma), version, &pReq) < 0) {
+      return -1;
     }
   }
 
-  tdUnRefSmaStat(pSma, pStat);
-
-  return TSDB_CODE_SUCCESS;
+  tdTSmaAdd(pSma, 1);
+  return 0;
 }
\ No newline at end of file

source/dnode/vnode/src/tq/tq.c

@@ -130,7 +130,17 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
           TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset);
 
   STqHandle* pHandle = taosHashGet(pTq->handles, pReq->subKey, strlen(pReq->subKey));
-  ASSERT(pHandle);
+  /*ASSERT(pHandle);*/
+  if (pHandle == NULL) {
+    tqError("tmq poll: no consumer handle for consumer %ld in vg %d, subkey %s", consumerId, pTq->pVnode->config.vgId,
+            pReq->subKey);
+    return -1;
+  }
+  if (pHandle->consumerId != consumerId) {
+    tqError("tmq poll: consumer handle mismatch for consumer %ld in vg %d, subkey %s, handle consumer id %ld",
+            consumerId, pTq->pVnode->config.vgId, pReq->subKey, pHandle->consumerId);
+    return -1;
+  }
 
   int32_t consumerEpoch = atomic_load_32(&pHandle->epoch);
   while (consumerEpoch < reqEpoch) {

source/dnode/vnode/src/tq/tqPush.c

@@ -238,9 +238,6 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
   if (msgType == TDMT_VND_SUBMIT) {
     if (taosHashGetSize(pTq->pStreamTasks) == 0) return 0;
 
-    if (tdUpdateExpireWindow(pTq->pVnode->pSma, msg, ver) != 0) {
-      // TODO handle sma error
-    }
     void* data = taosMemoryMalloc(msgLen);
     if (data == NULL) {
       return -1;

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -150,7 +150,7 @@ int32_t tsdbCommit(STsdb *pTsdb) {
   return code;
 
 _err:
-  tsdbError("vgId:%d failed to commit since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d, failed to commit since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
   return code;
 }
 

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -176,13 +176,13 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid
 
   pMemTable->nDelOp++;
 
-  tsdbError("vgId:%d delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64
+  tsdbError("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64
             " since %s",
             TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code));
   return code;
 
 _err:
-  tsdbError("vgId:%d failed to delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64
+  tsdbError("vgId:%d, failed to delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64
             " since %s",
             TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code));
   return code;

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -225,6 +225,7 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
   vTrace("message in fetch queue is processing");
   char   *msgstr = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
   int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
+
   switch (pMsg->msgType) {
     case TDMT_VND_FETCH:
       return qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg, 0);
@@ -236,13 +237,10 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
       return qWorkerProcessDropMsg(pVnode, pVnode->pQuery, pMsg, 0);
     case TDMT_VND_QUERY_HEARTBEAT:
       return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg, 0);
-
     case TDMT_VND_TABLE_META:
       return vnodeGetTableMeta(pVnode, pMsg);
-
     case TDMT_VND_CONSUME:
       return tqProcessPollReq(pVnode->pTq, pMsg, pInfo->workerId);
-
     case TDMT_STREAM_TASK_RUN:
       return tqProcessTaskRunReq(pVnode->pTq, pMsg);
     case TDMT_STREAM_TASK_DISPATCH:
@@ -279,7 +277,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, int64_t version, SRpcMsg *pMsg, SRp
 void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) {
   // TODO
 
-  // blockDebugShowData(data);
+  // blockDebugShowData(data, __func__);
   tdProcessTSmaInsert(((SVnode *)pVnode)->pSma, smaId, (const char *)data);
 }
 

source/dnode/vnode/test/tsdbSmaTest.cpp

@@ -373,7 +373,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
   pTsdb->pTfs = tfsOpen(&pDisks, numOfDisks);
   EXPECT_NE(pTsdb->pTfs, nullptr);
 
-  // generate SSubmitReq msg and update expired window
+  // generate SSubmitReq msg and update expire window
   int16_t  schemaVer = 0;
   uint32_t mockRowLen = sizeof(STSRow);
   uint32_t mockRowNum = 2;

source/libs/executor/inc/executorimpl.h

@@ -759,7 +759,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scan
 int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz);
 
 void    doSetOperatorCompleted(SOperatorInfo* pOperator);
-void    doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, bool needFree);
+void    doFilter(const SNode* pFilterNode, SSDataBlock* pBlock);
 SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowCellInfoOffset);
 void    relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols);
 void    initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow);

source/libs/executor/src/executorimpl.c

@@ -1819,9 +1819,9 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO
   }
 }
 
-static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep, bool needFree);
+static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep);
 
-void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, bool needFree) {
+void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) {
   if (pFilterNode == NULL) {
     return;
   }
@@ -1840,30 +1840,29 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, bool needFree) {
   bool keep = filterExecute(filter, pBlock, &rowRes, NULL, param1.numOfCols);
   filterFreeInfo(filter);
 
-  extractQualifiedTupleByFilterResult(pBlock, rowRes, keep, needFree);
+  extractQualifiedTupleByFilterResult(pBlock, rowRes, keep);
   blockDataUpdateTsWindow(pBlock, 0);
 }
 
-void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep, bool needFree) {
+void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep) {
   if (keep) {
     return;
   }
 
   if (rowRes != NULL) {
-    SSDataBlock* px = createOneDataBlock(pBlock, false);
-    blockDataEnsureCapacity(px, pBlock->info.rows);
+    SSDataBlock* px = createOneDataBlock(pBlock, true);
 
     int32_t totalRows = pBlock->info.rows;
-
     for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
-      SColumnInfoData* pDst = taosArrayGet(px->pDataBlock, i);
-      SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, i);
-
+      SColumnInfoData* pSrc = taosArrayGet(px->pDataBlock, i);
+      SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i);
       // it is a reserved column for scalar function, and no data in this column yet.
-      if (pSrc->pData == NULL) {
+      if (pDst->pData == NULL) {
         continue;
       }
 
+      colInfoDataCleanup(pDst, pBlock->info.rows);
+
       int32_t numOfRows = 0;
       for (int32_t j = 0; j < totalRows; ++j) {
         if (rowRes[j] == 0) {
@@ -1883,20 +1882,8 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR
       } else {
         ASSERT(pBlock->info.rows == numOfRows);
       }
-
-      SColumnInfoData tmp = *pSrc;
-      *pSrc = *pDst;
-      *pDst = tmp;
-
-      if (!needFree) {
-        if (IS_VAR_DATA_TYPE(pDst->info.type)) {  // this elements do not need free
-          pDst->varmeta.offset = NULL;
-        } else {
-          pDst->nullbitmap = NULL;
-        }
-        pDst->pData = NULL;
-      }
     }
+
     blockDataDestroy(px);  // fix memory leak
   } else {
     // do nothing
@@ -2137,11 +2124,6 @@ static int32_t compressQueryColData(SColumnInfoData* pColRes, int32_t numOfRows,
 }
 
 int32_t doFillTimeIntervalGapsInResults(struct SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t capacity) {
-  //  for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
-  //    SColumnInfoData* pColInfoData = taosArrayGet(pOutput->pDataBlock, i);
-  //    p[i] = pColInfoData->pData + (pColInfoData->info.bytes * pOutput->info.rows);
-  //  }
-
   int32_t numOfRows = (int32_t)taosFillResultDataBlock(pFillInfo, pBlock, capacity - pBlock->info.rows);
   pBlock->info.rows += numOfRows;
 
@@ -2896,7 +2878,6 @@ static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) {
     return seqLoadRemoteData(pOperator);
   } else {
     return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo);
-    //    return concurrentlyLoadRemoteData(pOperator);
   }
 }
 
@@ -2922,9 +2903,14 @@ static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo* pInfo) {
+static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo* pInfo, const char* id) {
   size_t numOfSources = LIST_LENGTH(pExNode->pSrcEndPoints);
 
+  if (numOfSources == 0) {
+    qError("%s invalid number: %d of sources in exchange operator", id, (int32_t) numOfSources);
+    return TSDB_CODE_INVALID_PARA;
+  }
+
   pInfo->pSources = taosArrayInit(numOfSources, sizeof(SDownstreamSourceNode));
   pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo));
   if (pInfo->pSourceDataInfo == NULL || pInfo->pSources == NULL) {
@@ -2946,7 +2932,7 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode
     goto _error;
   }
 
-  int32_t code = initExchangeOperator(pExNode, pInfo);
+  int32_t code = initExchangeOperator(pExNode, pInfo, GET_TASKID(pTaskInfo));
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -2975,7 +2961,7 @@ _error:
 
   taosMemoryFreeClear(pInfo);
   taosMemoryFreeClear(pOperator);
-  pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
+  pTaskInfo->code = code;
   return NULL;
 }
 
@@ -3744,7 +3730,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
       longjmp(pTaskInfo->env, code);
     }
 
-    doFilter(pProjectInfo->pFilterNode, pBlock, true);
+    doFilter(pProjectInfo->pFilterNode, pBlock);
 
     setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, scanFlag, false);
     blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows);
@@ -5274,7 +5260,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
   (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId,
                                            &(*pTaskInfo)->tableqinfoList, pPlan->pTagCond);
   if (NULL == (*pTaskInfo)->pRoot) {
-    code = terrno;
+    code = (*pTaskInfo)->code;
     goto _complete;
   }
 
@@ -5287,7 +5273,6 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
 
 _complete:
   taosMemoryFreeClear(*pTaskInfo);
-
   terrno = code;
   return code;
 }

source/libs/executor/src/groupoperator.c

@@ -359,7 +359,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
 
   while(1) {
     doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
-    doFilter(pInfo->pCondition, pRes, true);
+    doFilter(pInfo->pCondition, pRes);
 
     bool hasRemain = hashRemainDataInGroupInfo(&pInfo->groupResInfo);
     if (!hasRemain) {

source/libs/executor/src/scanoperator.c

@@ -267,7 +267,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
   }
 
   int64_t st = taosGetTimestampMs();
-  doFilter(pTableScanInfo->pFilterNode, pBlock, false);
+  doFilter(pTableScanInfo->pFilterNode, pBlock);
 
   int64_t et = taosGetTimestampMs();
   pTableScanInfo->readRecorder.filterTime += (et - st);
@@ -948,7 +948,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
         addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes);
       }
 
-      doFilter(pInfo->pCondition, pInfo->pRes, false);
+      doFilter(pInfo->pCondition, pInfo->pRes);
       blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);
       break;
     }
@@ -1716,7 +1716,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
   }
 
   pRes->info.rows = count;
-  doFilter(pInfo->pFilterNode, pRes, true);
+  doFilter(pInfo->pFilterNode, pRes);
 
   pOperator->resultInfo.totalRows += pRes->info.rows;
 

source/libs/executor/src/sortoperator.c

@@ -296,7 +296,10 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) {
 }
 
 SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity,
-                                        SArray* pColMatchInfo, SMultiwaySortMergeOperatorInfo* pInfo) {
+                                        SArray* pColMatchInfo, SOperatorInfo* pOperator) {
+  SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info;
+  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+
   blockDataCleanup(pDataBlock);
 
   SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
@@ -354,6 +357,8 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
   }
 
   blockDataDestroy(p);
+
+  qDebug("%s get sorted row blocks, rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.rows);
   return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
 }
 
@@ -371,7 +376,7 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
   }
 
   SSDataBlock* pBlock = getMultiwaySortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes,
-                                                   pOperator->resultInfo.capacity, pInfo->pColMatchInfo, pInfo);
+                                                   pOperator->resultInfo.capacity, pInfo->pColMatchInfo, pOperator);
 
   if (pBlock != NULL) {
     pOperator->resultInfo.totalRows += pBlock->info.rows;

source/libs/executor/src/timewindowoperator.c

@@ -750,14 +750,15 @@ int64_t getReskey(void* data, int32_t index) {
   return *(int64_t*)pos->key;
 }
 
-static int32_t saveResult(SResultRow* result, uint64_t groupId, SArray* pUpdated) {
+static int32_t saveResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId,
+    SArray* pUpdated) {
   int32_t size = taosArrayGetSize(pUpdated);
-  int32_t index = binarySearch(pUpdated, size, result->win.skey, TSDB_ORDER_DESC, getReskey);
+  int32_t index = binarySearch(pUpdated, size, ts, TSDB_ORDER_DESC, getReskey);
   if (index == -1) {
     index = 0;
   } else {
     TSKEY resTs = getReskey(pUpdated, index);
-    if (resTs < result->win.skey) {
+    if (resTs < ts) {
       index++;
     } else {
       return TSDB_CODE_SUCCESS;
@@ -769,14 +770,18 @@ static int32_t saveResult(SResultRow* result, uint64_t groupId, SArray* pUpdated
     return TSDB_CODE_OUT_OF_MEMORY;
   }
   newPos->groupId = groupId;
-  newPos->pos = (SResultRowPosition){.pageId = result->pageId, .offset = result->offset};
-  *(int64_t*)newPos->key = result->win.skey;
+  newPos->pos = (SResultRowPosition){.pageId = pageId, .offset = offset};
+  *(int64_t*)newPos->key = ts;
   if (taosArrayInsert(pUpdated, index, &newPos) == NULL) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t saveResultRow(SResultRow* result, uint64_t groupId, SArray* pUpdated) {
+  return saveResult(result->win.skey, result->pageId, result->offset, groupId, pUpdated);
+}
+
 static void removeResult(SArray* pUpdated, TSKEY key) {
   int32_t size = taosArrayGetSize(pUpdated);
   int32_t index = binarySearch(pUpdated, size, key, TSDB_ORDER_DESC, getReskey);
@@ -819,7 +824,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
 
   if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
     if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
-      saveResult(pResult, tableGroupId, pUpdated);
+      saveResultRow(pResult, tableGroupId, pUpdated);
     }
   }
 
@@ -869,7 +874,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
 
     if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
       if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
-        saveResult(pResult, tableGroupId, pUpdated);
+        saveResultRow(pResult, tableGroupId, pUpdated);
       }
     }
 
@@ -1243,6 +1248,23 @@ static void doClearWindows(SAggSupporter* pSup, SOptrBasicInfo* pBinfo, SInterva
   }
 }
 
+static int32_t getAllIntervalWindow(SHashObj* pHashMap, SArray* resWins) {
+  void*  pIte = NULL;
+  size_t keyLen = 0;
+  while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) {
+    void*    key = taosHashGetKey(pIte, &keyLen);
+    uint64_t groupId = *(uint64_t*)key;
+    ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY)));
+    TSKEY          ts = *(int64_t*)((char*)key + sizeof(uint64_t));
+    SResultRowPosition* pPos = (SResultRowPosition*)pIte;
+    int32_t code = saveResult(ts, pPos->pageId, pPos->offset, groupId, resWins);
+    if (code != TSDB_CODE_SUCCESS) {
+      return code;
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval,
                                    SArray* closeWins) {
   void*  pIte = NULL;
@@ -1259,16 +1281,12 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup,
       char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))];
       SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId);
       taosHashRemove(pHashMap, keyBuf, keyLen);
-      SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
-      if (pos == NULL) {
-        return TSDB_CODE_OUT_OF_MEMORY;
-      }
-      pos->groupId = groupId;
-      pos->pos = *(SResultRowPosition*)pIte;
-      *(int64_t*)pos->key = ts;
-      if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE && !taosArrayPush(closeWins, &pos)) {
-        taosMemoryFree(pos);
-        return TSDB_CODE_OUT_OF_MEMORY;
+      SResultRowPosition* pPos = (SResultRowPosition*)pIte;
+      if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
+        int32_t code = saveResult(ts, pPos->pageId, pPos->offset, groupId, closeWins);
+        if (code != TSDB_CODE_SUCCESS) {
+          return code;
+        }
       }
     }
   }
@@ -1296,8 +1314,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
   SOperatorInfo* downstream = pOperator->pDownstream[0];
 
   SArray* pUpdated = taosArrayInit(4, POINTER_BYTES);
-  SArray* pClosed = taosArrayInit(4, POINTER_BYTES);
-
   while (1) {
     SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
     if (pBlock == NULL) {
@@ -1316,19 +1332,18 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
       doClearWindows(&pInfo->aggSup, &pInfo->binfo, &pInfo->interval, 0, pOperator->numOfExprs, pBlock, NULL);
       qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo));
       continue;
+    } else if (pBlock->info.type == STREAM_GET_ALL &&
+        pInfo->twAggSup.calTrigger == STREAM_TRIGGER_MAX_DELAY) {
+      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated);
+      continue;
     }
 
     pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
     hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdated);
   }
+  closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pUpdated);
 
-  closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pClosed);
-  finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pClosed, pInfo->binfo.rowCellInfoOffset);
-  taosArrayAddAll(pUpdated, pClosed);
-
-  taosArrayDestroy(pClosed);
   finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset);
-
   initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
   blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
   doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
@@ -1898,7 +1913,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc
     forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL,
                                            TSDB_ORDER_ASC);
     if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdated) {
-      saveResult(pResult, tableGroupId, pUpdated);
+      saveResultRow(pResult, tableGroupId, pUpdated);
     }
     // window start(end) key interpolation
     // doWindowBorderInterpolation(pInfo, pSDataBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos,
@@ -1993,7 +2008,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
   SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info;
   SOperatorInfo*                    downstream = pOperator->pDownstream[0];
   SArray*                           pUpdated = taosArrayInit(4, POINTER_BYTES);
-  SArray*                           pClosed = taosArrayInit(4, POINTER_BYTES);
 
   if (pOperator->status == OP_EXEC_DONE) {
     return NULL;
@@ -2042,7 +2056,12 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
       copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex);
       taosArrayDestroy(pUpWins);
       break;
+    } else if (pBlock->info.type == STREAM_GET_ALL && isFinalInterval(pInfo) &&
+        pInfo->twAggSup.calTrigger == STREAM_TRIGGER_MAX_DELAY) {
+      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated);
+      continue;
     }
+
     if (isFinalInterval(pInfo)) {
       int32_t chIndex = getChildIndex(pBlock);
       int32_t size = taosArrayGetSize(pInfo->pChildren);
@@ -2064,13 +2083,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
   }
 
   if (isFinalInterval(pInfo)) {
-    closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pClosed);
-    finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pClosed, pInfo->binfo.rowCellInfoOffset);
-    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
-      taosArrayAddAll(pUpdated, pClosed);
-    }
+    closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pUpdated);
   }
-  taosArrayDestroy(pClosed);
 
   finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset);
   initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);

source/libs/executor/src/tsort.c

@@ -227,6 +227,8 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int
     for (int32_t i = 0; i < cmpParam->numOfSources; ++i) {
       SSortSource* pSource = cmpParam->pSources[i];
       pSource->src.pBlock = pHandle->fetchfp(pSource->param);
+
+      // set current source id done
       if (pSource->src.pBlock == NULL) {
         pSource->src.rowIndex = -1;
         ++pHandle->numOfCompletedSources;
@@ -426,8 +428,16 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
   double sortPass = floorl(log2(numOfSources) / log2(pHandle->numOfPages));
 
   pHandle->totalElapsed = taosGetTimestampUs() - pHandle->startTs;
-  qDebug("%s %d rounds mergesort required to complete the sort, first-round sorted data size:%"PRIzu", sort elapsed:%"PRId64", total elapsed:%"PRId64,
-         pHandle->idStr, (int32_t) (sortPass + 1), pHandle->pBuf ? getTotalBufSize(pHandle->pBuf) : 0, pHandle->sortElapsed, pHandle->totalElapsed);
+
+  if (sortPass > 0) {
+    size_t s = pHandle->pBuf ? getTotalBufSize(pHandle->pBuf) : 0;
+    qDebug("%s %d rounds mergesort required to complete the sort, first-round sorted data size:%" PRIzu
+           ", sort elapsed:%" PRId64 ", total elapsed:%" PRId64,
+           pHandle->idStr, (int32_t)(sortPass + 1), s, pHandle->sortElapsed, pHandle->totalElapsed);
+  } else {
+    qDebug("%s ordered source:%"PRIzu", available buf:%d, no need internal sort", pHandle->idStr, numOfSources,
+        pHandle->numOfPages);
+  }
 
   int32_t numOfRows = blockDataGetCapacityInRow(pHandle->pDataBlock, pHandle->pageSize);
   blockDataEnsureCapacity(pHandle->pDataBlock, numOfRows);

source/libs/parser/src/parser.c

@@ -76,7 +76,7 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) {
   int32_t inputSize = (NULL != pParam->length ? *(pParam->length) : tDataTypes[pParam->buffer_type].bytes);
   pVal->node.resType.type = pParam->buffer_type;
   pVal->node.resType.bytes = inputSize;
-  
+
   switch (pParam->buffer_type) {
     case TSDB_DATA_TYPE_VARCHAR:
     case TSDB_DATA_TYPE_VARBINARY:
@@ -186,7 +186,7 @@ int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** p
   return extractResultSchema(pRoot, numOfCols, pSchema);
 }
 
-int32_t qSetSTableIdForRSma(SNode* pStmt, int64_t uid) {
+int32_t qSetSTableIdForRsma(SNode* pStmt, int64_t uid) {
   if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) {
     SNode* pTable = ((SSelectStmt*)pStmt)->pFromTable;
     if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) {

source/libs/stream/src/streamDispatch.c

@@ -100,7 +100,6 @@ int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcM
       .upstreamNodeId = pTask->nodeId,
       .blockNum = blockNum,
   };
-  qInfo("dispatch from task %d (child id %d)", pTask->taskId, pTask->childId);
 
   req.data = taosArrayInit(blockNum, sizeof(void*));
   req.dataLen = taosArrayInit(blockNum, sizeof(int32_t));
@@ -142,11 +141,14 @@ int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcM
         break;
       }
     }
-    ASSERT(vgId != 0);
   }
 
+  ASSERT(vgId != 0);
   req.taskId = downstreamTaskId;
 
+  qInfo("dispatch from task %d (child id %d) to down stream task %d in vnode %d", pTask->taskId, pTask->childId,
+        downstreamTaskId, vgId);
+
   // serialize
   int32_t tlen;
   tEncodeSize(tEncodeStreamDispatchReq, &req, tlen, code);

source/util/src/terror.c

@@ -353,9 +353,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVLD_TAG_VAL,             "TSDB invalid tag valu
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_CACHE_LAST_ROW,        "TSDB no cache last row data")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECREATED,          "Table re-created")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR,       "TDB env open error")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_SMA_INDEX_IN_META,     "No sma index in meta")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_SMA_STAT,         "Invalid sma state")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TSMA_ALREADY_EXIST,       "TSMA already exists")
 
 
 // query
@@ -537,25 +534,38 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_DELETE_WHERE,       "The DELETE statemen
 TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_REDISTRIBUTE_VG,    "The REDISTRIBUTE VGROUP statement only support 1 to 3 dnodes")
 
 //planner
-TAOS_DEFINE_ERROR(TSDB_CODE_PLAN_INTERNAL_ERROR,          "Planner internal error")
+TAOS_DEFINE_ERROR(TSDB_CODE_PLAN_INTERNAL_ERROR,            "Planner internal error")
 
 //udf
-TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING,                 "udf is stopping")
-TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_READ_ERR,            "udf pipe read error")
-TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_CONNECT_ERR,         "udf pipe connect error")
-TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_NO_PIPE,             "udf no pipe")
-TAOS_DEFINE_ERROR(TSDB_CODE_UDF_LOAD_UDF_FAILURE,         "udf load failure")
-TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_STATE,            "udf invalid state")
-TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_INPUT,            "udf invalid function input")
-TAOS_DEFINE_ERROR(TSDB_CODE_UDF_NO_FUNC_HANDLE,           "udf no function handle")
-TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_BUFSIZE,          "udf invalid bufsize")
-TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_OUTPUT_TYPE,      "udf invalid output type")
+TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING,                   "udf is stopping")
+TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_READ_ERR,              "udf pipe read error")
+TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_CONNECT_ERR,           "udf pipe connect error")
+TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_NO_PIPE,               "udf no pipe")
+TAOS_DEFINE_ERROR(TSDB_CODE_UDF_LOAD_UDF_FAILURE,           "udf load failure")
+TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_STATE,              "udf invalid state")
+TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_INPUT,              "udf invalid function input")
+TAOS_DEFINE_ERROR(TSDB_CODE_UDF_NO_FUNC_HANDLE,             "udf no function handle")
+TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_BUFSIZE,            "udf invalid bufsize")
+TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_OUTPUT_TYPE,        "udf invalid output type")
 
 //schemaless
-TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE,    "Invalid line protocol type")
-TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PRECISION_TYPE,   "Invalid timestamp precision type")
-TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DATA,             "Invalid data type")
-TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DB_CONF,          "Invalid schemaless db config")
+TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE,      "Invalid line protocol type")
+TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PRECISION_TYPE,     "Invalid timestamp precision type")
+TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DATA,               "Invalid data type")
+TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DB_CONF,            "Invalid schemaless db config")
+
+//tsma
+TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_ALREADY_EXIST,             "Tsma already exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_NO_INDEX_IN_META,          "No tsma index in meta")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_INVALID_ENV,               "Invalid tsma env")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_INVALID_STAT,              "Invalid tsma state")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_NO_INDEX_IN_CACHE,         "No tsma index in cache")
+TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_RM_SKEY_IN_HASH,           "Rm tsma skey in cache")
+
+//rsma
+TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_ENV,               "Invalid rsma env")
+TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_STAT,              "Invalid rsma state")
+
 
 #ifdef TAOS_ERROR_C
 };

tests/script/jenkins/basic.txt

@@ -57,7 +57,7 @@
 
 # ---- mnode
 ./test.sh -f tsim/mnode/basic1.sim
-#./test.sh -f tsim/mnode/basic2.sim
+./test.sh -f tsim/mnode/basic2.sim
 ./test.sh -f tsim/mnode/basic3.sim
 ./test.sh -f tsim/mnode/basic4.sim
 

tests/script/tsim/mnode/basic2.sim

@@ -92,6 +92,8 @@ sql show mnodes
 if $rows != 2 then 
   return -1
 endi
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
 
 sql show users
 if $rows != 2 then 
@@ -111,6 +113,8 @@ step3:
 		return -1
 	endi
 sql show dnodes -x step3
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
 if $data(1)[4] != ready then
   goto step3
 endi

tests/script/tsim/mnode/basic3.sim

@@ -39,11 +39,9 @@ endi
 print =============== step2: create mnode 2
 sql create mnode on dnode 2
 sql create mnode on dnode 3
-return
-system sh/exec.sh -n dnode1 -s stop -x SIGKILL
 sql_error create mnode on dnode 4
 
-
+$leaderExist = 0
 $x = 0
 step2: 
 	$x = $x + 1
@@ -52,13 +50,20 @@ step2:
 		return -1
 	endi
 sql show mnodes -x step2
-if $data(1)[2] != leader then
-  goto step2
+
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+if $data(1)[2] == leader then
+  $leaderExist = 1
 endi
-if $data(2)[2] != follower then
-  goto step2
+if $data(2)[2] == leader then
+  $leaderExist = 1
+endi
+if $data(3)[2] == leader then
+  $leaderExist = 1
 endi
-if $data(3)[2] != follower then
+if  $leaderExist != 1 then
   goto step2
 endi
 
@@ -70,10 +75,10 @@ if $rows != 2 then
 endi
 
 # wait mnode2 mnode3 recv data finish
-sleep 10000
+sleep 1000
 
 print =============== step4: stop dnode1
-system sh/exec.sh -n dnode1 -s stop
+system sh/exec.sh -n dnode1 -s stop -x SIGKILL
 
 $x = 0
 step4: 
@@ -92,13 +97,22 @@ if $rows != 2 then
   return -1
 endi
 
-sleep 1000
-sql show dnodes
+$x = 0
+step41: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+sql show dnodes -x step41
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
 if $data(2)[4] != ready then
-  return -1
+  goto step41
 endi
 if $data(3)[4] != ready then
-  return -1
+  goto step41
 endi
 
 print =============== step5: stop dnode1
@@ -117,15 +131,29 @@ print $data(1)[0] $data(1)[1] $data(1)[2]
 print $data(2)[0] $data(2)[1] $data(2)[2] 
 print $data(3)[0] $data(3)[1] $data(3)[2] 
 
-if $data(2)[2] != offline then
-  goto step5
-endi
-
 sql show users
 if $rows != 2 then 
   return -1
 endi
 
+$x = 0
+step51: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+sql show dnodes -x step51
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+if $data(1)[4] != ready then
+  goto step51
+endi
+if $data(3)[4] != ready then
+  goto step51
+endi
+
 print =============== step6: stop dnode1
 system sh/exec.sh -n dnode2 -s start
 system sh/exec.sh -n dnode3 -s stop
@@ -147,6 +175,24 @@ if $rows != 2 then
   return -1
 endi
 
+$x = 0
+step61: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+sql show dnodes -x step61
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+if $data(1)[4] != ready then
+  goto step61
+endi
+if $data(2)[4] != ready then
+  goto step61
+endi
+
 system sh/exec.sh -n dnode1 -s stop
 system sh/exec.sh -n dnode2 -s stop
 system sh/exec.sh -n dnode3 -s stop

tests/system-test/1-insert/alter_stable.py

@@ -122,10 +122,10 @@ class TDTestCase:
 
         for i in ['bigint','unsigned int','float','double','binary(10)','nchar(10)']:
             for j in [1,2,3]:
-                tdSql.error(f'alter table {stbname} modify tag t{j} {i}')
+                tdSql.error(f'alter stable {stbname} modify tag t{j} {i}')
         for i in ['int','unsigned int','float','binary(10)','nchar(10)']:
-            tdSql.error(f'alter table {stbname} modify tag t8 {i}')
-        tdSql.error(f'alter table {stbname} modify tag t4 int')
+            tdSql.error(f'alter stable {stbname} modify tag t8 {i}')
+        tdSql.error(f'alter stable {stbname} modify tag t4 int')
         tdSql.execute(f'drop database {dbname}')
 
     def run(self):

tests/system-test/1-insert/alter_table.py

@@ -145,10 +145,17 @@ class TDTestCase:
         tdSql.execute(f'alter table {dbname}.{tbname} rename column c1 c21')
         tdSql.query(f'describe {dbname}.{tbname}')
         tdSql.checkData(1,0,'c21')
+        # !bug TD-16423
+        # tdSql.error(f'select c1 from {dbname}.{tbname}')
+        # tdSql.query(f'select c21 from {dbname}.{tbname}')
+        # tdSql.checkData(0,1,1)
         tdSql.execute(f'alter table {dbname}.{tbname} rename column `c21` c1')
         tdSql.query(f'describe {dbname}.{tbname}')
         tdSql.checkData(1,0,'c1')
-
+        # !bug TD-16423
+        # tdSql.error(f'select c1 from {dbname}.{tbname}')
+        # tdSql.query(f'select c1 from {dbname}.{tbname}')
+        # tdSql.checkData(0,1,1)
         tdSql.error(f'alter table {dbname}.{tbname} modify column c1 bigint')
         tdSql.error(f'alter table {dbname}.{tbname} modify column c1 double')
         tdSql.error(f'alter table {dbname}.{tbname} modify column c4 int')
@@ -158,9 +165,109 @@ class TDTestCase:
         tdSql.error(f'alter table {dbname}.{tbname} modify column c1 bool')
         tdSql.error(f'alter table {dbname}.{tbname} modify column c1 binary(10)')
         tdSql.execute(f'drop database {dbname}')
+    def alter_stb_column_check(self):
+        dbname = self.get_long_name(length=10, mode="letters")
+        tdSql.execute(f'create database if not exists {dbname}')
+        stbname = self.get_long_name(length=3, mode="letters")
+        tbname = self.get_long_name(length=3, mode="letters")
+        tdSql.execute(f'create database if not exists {dbname}')
+        tdSql.execute(f'use {dbname}')
+        tdSql.execute(
+            f'create table {stbname} (ts timestamp, c1 tinyint, c2 smallint, c3 int, \
+                c4 bigint, c5 tinyint unsigned, c6 smallint unsigned, c7 int unsigned, c8 bigint unsigned, c9 float, c10 double, c11 bool,c12 binary(20),c13 nchar(20)) tags(t0 int) ')
+        tdSql.execute(f'create table {tbname} using {stbname} tags(1)')
+        tdSql.execute(f'insert into {tbname} values (now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")')
+        tdSql.execute(f'alter table {stbname} add column c14 int')
+        tdSql.query(f'select c14 from {stbname}')
+        tdSql.checkRows(1)
+        tdSql.execute(f'alter table {stbname} add column `c15` int')
+        tdSql.query(f'select c15 from {stbname}')
+        tdSql.checkRows(1)
+        tdSql.query(f'describe {stbname}')
+        tdSql.checkRows(17)
+        tdSql.execute(f'alter table {stbname} drop column c14')
+        tdSql.query(f'describe {stbname}')
+        tdSql.checkRows(16)
+        tdSql.execute(f'alter table {stbname} drop column `c15`')
+        tdSql.query(f'describe {stbname}')
+        tdSql.checkRows(15)
+        tdSql.execute(f'alter table {stbname} modify column c12 binary(30)')
+        tdSql.query(f'describe {stbname}')
+        tdSql.checkData(12,2,30)
+        tdSql.execute(f'alter table {stbname} modify column `c12` binary(35)')
+        tdSql.query(f'describe {stbname}')
+        tdSql.checkData(12,2,35)
+        tdSql.error(f'alter table {stbname} modify column `c12` binary(34)')
+        tdSql.execute(f'alter table {stbname} modify column c13 nchar(30)')
+        tdSql.query(f'describe {stbname}')
+        tdSql.checkData(13,2,30)
+        tdSql.error(f'alter table {stbname} modify column c13 nchar(29)')
+        tdSql.error(f'alter table {stbname} rename column c1 c21')
+        tdSql.error(f'alter table {stbname} modify column c1 int')
+        tdSql.error(f'alter table {stbname} modify column c4 int')
+        tdSql.error(f'alter table {stbname} modify column c8 int')
+        tdSql.error(f'alter table {stbname} modify column c1 unsigned int')
+        tdSql.error(f'alter table {stbname} modify column c9 double')
+        tdSql.error(f'alter table {stbname} modify column c10 float')
+        tdSql.error(f'alter table {stbname} modify column c11 int')
+        tdSql.execute(f'drop database {dbname}')
+    def alter_stb_tag_check(self):
+        dbname = self.get_long_name(length=10, mode="letters")
+        tdSql.execute(f'create database if not exists {dbname}')
+        stbname = self.get_long_name(length=3, mode="letters")
+        tbname = self.get_long_name(length=3, mode="letters")
+        tdSql.execute(f'create database if not exists {dbname}')
+        tdSql.execute(f'use {dbname}')
+        tdSql.execute(
+            f'create table {stbname} (ts timestamp, c1 int) tags(ts_tag timestamp, t1 tinyint, t2 smallint, t3 int, \
+                t4 bigint, t5 tinyint unsigned, t6 smallint unsigned, t7 int unsigned, t8 bigint unsigned, t9 float, t10 double, t11 bool,t12 binary(20),t13 nchar(20)) ')
+        tdSql.execute(f'create table {tbname} using {stbname} tags(now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")')
+        tdSql.execute(f'insert into {tbname} values(now,1)')
+
+        tdSql.execute(f'alter table {stbname} add tag t14 int')
+        tdSql.query(f'select t14 from {stbname}')
+        tdSql.checkRows(1)
+        tdSql.execute(f'alter table {stbname} add tag `t15` int')
+        tdSql.query(f'select t14 from {stbname}')
+        tdSql.checkRows(1)
+        tdSql.query(f'describe {stbname}')
+        tdSql.checkRows(18)
+        tdSql.execute(f'alter table {stbname} drop tag t14')
+        tdSql.query(f'describe {stbname}')
+        tdSql.checkRows(17)
+        tdSql.execute(f'alter table {stbname} drop tag `t15`')
+        tdSql.query(f'describe {stbname}')
+        tdSql.checkRows(16)
+        tdSql.execute(f'alter table {stbname} modify tag t12 binary(30)')
+        tdSql.query(f'describe {stbname}')
+        tdSql.checkData(14,2,30)
+        tdSql.execute(f'alter table {stbname} modify tag `t12` binary(35)')
+        tdSql.query(f'describe {stbname}')
+        tdSql.checkData(14,2,35)
+        tdSql.error(f'alter table {stbname} modify tag `t12` binary(34)')
+        tdSql.execute(f'alter table {stbname} modify tag t13 nchar(30)')
+        tdSql.query(f'describe {stbname}')
+        tdSql.checkData(15,2,30)
+        tdSql.error(f'alter table {stbname} modify tag t13 nchar(29)')
+        tdSql.execute(f'alter table {stbname} rename tag t1 t21')
+        tdSql.query(f'describe {stbname}')
+        tdSql.checkData(3,0,'t21')
+        tdSql.execute(f'alter table {stbname} rename tag `t21` t1')
+        tdSql.query(f'describe {stbname}')
+        tdSql.checkData(3,0,'t1')
+
+        for i in ['bigint','unsigned int','float','double','binary(10)','nchar(10)']:
+            for j in [1,2,3]:
+                tdSql.error(f'alter table {stbname} modify tag t{j} {i}')
+        for i in ['int','unsigned int','float','binary(10)','nchar(10)']:
+            tdSql.error(f'alter table {stbname} modify tag t8 {i}')
+        tdSql.error(f'alter table {stbname} modify tag t4 int')
+        tdSql.execute(f'drop database {dbname}')
     def run(self):
         self.alter_tb_tag_check()
         self.alter_ntb_column_check()
+        self.alter_stb_column_check()
+        self.alter_stb_tag_check()
 
     def stop(self):
         tdSql.close()

tests/system-test/1-insert/influxdb_line_taosc_insert.py

@@ -23,6 +23,10 @@ from util.sql import *
 import threading
 from util.types import TDSmlProtocolType, TDSmlTimestampType
 from util.common import tdCom
+import platform
+import io
+if platform.system().lower() == 'windows':
+    sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf8') 
 
 class TDTestCase:
     def init(self, conn, logSql):

tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py

@@ -22,6 +22,10 @@ from util.sql import *
 from util.common import tdCom
 from util.types import TDSmlProtocolType, TDSmlTimestampType
 import threading
+import platform
+import io
+if platform.system().lower() == 'windows':
+    sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf8') 
 
 class TDTestCase:
     def init(self, conn, logSql):

tests/system-test/2-query/To_iso8601.py

@@ -3,7 +3,7 @@ from time import sleep
 from util.log import *
 from util.sql import *
 from util.cases import *
-
+import os
 
 
 
@@ -12,28 +12,47 @@ class TDTestCase:
     def init(self, conn, logSql):
         tdLog.debug(f"start to excute {__file__}")
         tdSql.init(conn.cursor())
-
-    def run(self):  # sourcery skip: extract-duplicate-method
-        tdSql.prepare()
-        # get system timezone
-        today_date = datetime.datetime.strptime(
-            datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d")
+        self.rowNum = 10
+        self.ts = 1640966400000  # 2022-1-1 00:00:00.000
+    def check_customize_param_ms(self):
+        
+        time_zone = os.popen('date "+%z"').read().strip()
+        tdSql.execute('create database db1 precision "ms"')
+        tdSql.execute('use db1')
+        tdSql.execute('create table if not exists ntb(ts timestamp, c1 int, c2 timestamp)')
+        for i in range(self.rowNum):
+            tdSql.execute("insert into ntb values(%d, %d, %d)" 
+                        % (self.ts + i, i + 1, self.ts + i))
+        tdSql.query('select to_iso8601(ts) from ntb')
+        for i in range(self.rowNum):
+            tdSql.checkEqual(tdSql.queryResult[i][0],f'2022-01-01T00:00:00.00{i}{time_zone}')
+            
+        timezone_list = ['+0000','+0100','+0200','+0300','+0330','+0400','+0500','+0530','+0600','+0700','+0800','+0900','+1000','+1100','+1200',\
+                        '+00','+01','+02','+03','+04','+05','+06','+07','+08','+09','+10','+11','+12',\
+                            '+00:00','+01:00','+02:00','+03:00','+03:30','+04:00','+05:00','+05:30','+06:00','+07:00','+08:00','+09:00','+10:00','+11:00','+12:00',\
+                            '-0000','-0100','-0200','-0300','-0400','-0500','-0600','-0700','-0800','-0900','-1000','-1100','-1200',\
+                        '-00','-01','-02','-03','-04','-05','-06','-07','-08','-09','-10','-11','-12',\
+                            '-00:00','-01:00','-02:00','-03:00','-04:00','-05:00','-06:00','-07:00','-08:00','-09:00','-10:00','-11:00','-12:00',\
+                                'z','Z']
+        for j in timezone_list:
+            tdSql.query(f'select to_iso8601(ts,"{j}") from ntb')
+            for i in range(self.rowNum):
+                tdSql.checkEqual(tdSql.queryResult[i][0],f'2022-01-01T00:00:00.00{i}{j}')
         
+        error_param_list = [0,100.5,'a','!']
+        for i in error_param_list:
+            tdSql.error(f'select to_iso8601(ts,"{i}") from ntb')
+        #! bug TD-16372:对于错误的时区，缺少校验
+        error_timezone_param = ['+13','-13','+1300','-1300','+0001','-0001','-0330','-0530']
+        for i in error_timezone_param:
+            tdSql.error(f'select to_iso8601(ts,"{i}") from ntb')
+        
+    def check_base_function(self):
+        tdSql.prepare()
         tdLog.printNoPrefix("==========step1:create tables==========")
-        tdSql.execute(
-            '''create table if not exists ntb
-            (ts timestamp, c1 int, c2 float,c3 double,c4 timestamp)
-            '''
-        )
-        tdSql.execute(
-            '''create table if not exists stb
-            (ts timestamp, c1 int, c2 float,c3 double,c4 timestamp) tags(t0 int)
-            '''
-        )
-        tdSql.execute(
-            '''create table if not exists stb_1 using stb tags(100)
-            '''
-        )
+        tdSql.execute('create table if not exists ntb(ts timestamp, c1 int, c2 float,c3 double,c4 timestamp)')
+        tdSql.execute('create table if not exists stb(ts timestamp, c1 int, c2 float,c3 double,c4 timestamp) tags(t0 int)')
+        tdSql.execute('create table if not exists stb_1 using stb tags(100)')
 
         tdLog.printNoPrefix("==========step2:insert data==========")
         tdSql.execute('insert into ntb values(now,1,1.55,100.555555,today())("2020-1-1 00:00:00",10,11.11,99.999999,now())(today(),3,3.333,333.333333,now())')
@@ -48,12 +67,9 @@ class TDTestCase:
         tdSql.checkRows(1)
         tdSql.query("select to_iso8601(ts) from ntb where ts=today()")
         tdSql.checkRows(1)
-        # tdSql.checkData(0,0,10)
-        for i in range(1,10):
+        for i in range(0,3):
             tdSql.query("select to_iso8601(1) from ntb")
-            tdSql.checkData(0,0,"1970-01-01T08:00:01+0800")
-            i+=1
-            sleep(0.2)
+            tdSql.checkData(i,0,"1970-01-01T08:00:01+0800")
             tdSql.checkRows(3)
         tdSql.query("select to_iso8601(ts) from ntb")
         tdSql.checkRows(3)
@@ -67,53 +83,34 @@ class TDTestCase:
         tdSql.error("select to_iso8601(timezone()) from ntb")
         tdSql.error("select to_iso8601('abc') from ntb")
 
-        tdSql.query("select to_iso8601(today()) *null from ntb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,None)
-        tdSql.query("select to_iso8601(today()) +null from ntb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,None)
-        tdSql.query("select to_iso8601(today()) -null from ntb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,None)
-        tdSql.query("select to_iso8601(today()) /null from ntb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,None)
-
-        tdSql.query("select to_iso8601(today()) *null from db.ntb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,None)
-        tdSql.query("select to_iso8601(today()) +null from db.ntb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,None)
-        tdSql.query("select to_iso8601(today()) -null from db.ntb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,None)
-        tdSql.query("select to_iso8601(today()) /null from db.ntb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,None)
-        # tdSql.query("select to_iso8601(-1) from ntb")
+        for i in ['+','-','*','/']:
+            tdSql.query(f"select to_iso8601(today()) {i}null from ntb")
+            tdSql.checkRows(3)
+            tdSql.checkData(0,0,None)
+            tdSql.query(f"select to_iso8601(today()) {i}null from db.ntb")
+            tdSql.checkRows(3)
+            tdSql.checkData(0,0,None)
         tdSql.query("select to_iso8601(9223372036854775807) from ntb")
         tdSql.checkRows(3)
-        # bug TD-14896
+        # bug TD-15207
         # tdSql.query("select to_iso8601(10000000000) from ntb")
         # tdSql.checkData(0,0,None)
         # tdSql.query("select to_iso8601(-1) from ntb")
         # tdSql.checkRows(3)
         # tdSql.query("select to_iso8601(-10000000000) from ntb")
         # tdSql.checkData(0,0,None)
-        tdSql.error("select to_iso8601(1.5) from ntb")
-        tdSql.error("select to_iso8601(1.5) from db.ntb")
-        tdSql.error("select to_iso8601('a') from ntb")
-        tdSql.error("select to_iso8601(c2) from ntb")
+        err_param = [1.5,'a','c2']
+        for i in err_param:
+            tdSql.error(f"select to_iso8601({i}) from ntb")
+            tdSql.error(f"select to_iso8601({i}) from db.ntb")
+            
         tdSql.query("select to_iso8601(now) from stb")
+        tdSql.checkRows(3)
         tdSql.query("select to_iso8601(now()) from stb")
         tdSql.checkRows(3)
-        for i in range(1,10):
-            tdSql.query("select to_iso8601(1) from stb")
-            tdSql.checkData(0,0,"1970-01-01T08:00:01+0800")
-            i+=1
-            sleep(0.2)
+        tdSql.query("select to_iso8601(1) from stb")
+        for i in range(0,3):
+            tdSql.checkData(i,0,"1970-01-01T08:00:01+0800")
             tdSql.checkRows(3)
         tdSql.query("select to_iso8601(ts) from stb")
         tdSql.checkRows(3)
@@ -121,37 +118,17 @@ class TDTestCase:
         tdSql.checkRows(3)
         tdSql.query("select to_iso8601(ts)+'a' from stb ")
         tdSql.checkRows(3)
+        for i in ['+','-','*','/']:
+            tdSql.query(f"select to_iso8601(today()) {i}null from stb")
+            tdSql.checkRows(3)
+            tdSql.checkData(0,0,None)
+            tdSql.query(f"select to_iso8601(today()) {i}null from db.stb")
+            tdSql.checkRows(3)
+            tdSql.checkData(0,0,None)
         
-        tdSql.query("select to_iso8601(today()) *null from stb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,None)
-        tdSql.query("select to_iso8601(today()) +null from stb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,None)
-        tdSql.query("select to_iso8601(today()) -null from stb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,None)
-        tdSql.query("select to_iso8601(today()) /null from stb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,None)
-        tdSql.query("select to_iso8601(today()) *null from db.stb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,None)
-        tdSql.query("select to_iso8601(today()) +null from db.stb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,None)
-        tdSql.query("select to_iso8601(today()) -null from db.stb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,None)
-        tdSql.query("select to_iso8601(today()) /null from db.stb")
-        tdSql.checkRows(3)
-        tdSql.checkData(0,0,None)
-
-        # bug TD-14896
-        # tdSql.query("select to_iso8601(-1) from ntb")
-        # tdSql.checkRows(3)
-
-
+    def run(self):  # sourcery skip: extract-duplicate-method
+        self.check_base_function()
+        self.check_customize_param_ms()
 
     def stop(self):
         tdSql.close()

tests/system-test/2-query/abs.py

@@ -65,6 +65,60 @@ class TDTestCase:
             '''
         )
     
+
+    def prepare_tag_datas(self):
+                # prepare datas
+        tdSql.execute("create database if not exists testdb keep 3650 days 1000")
+        tdSql.execute(" use testdb ")
+        tdSql.execute(
+            '''create table stb1
+            (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+            tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32))
+            '''
+        )
+
+        tdSql.execute(
+            '''
+            create table t1
+            (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+            '''
+        )
+        for i in range(4):
+            tdSql.execute(f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )')
+
+        for i in range(9):
+            tdSql.execute(
+                f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
+            )
+            tdSql.execute(
+                f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )"
+            )
+        tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )")
+        tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
+        tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )")
+        tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )")
+
+        tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
+        tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
+        tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL  ) ")
+
+        tdSql.execute(
+            f'''insert into t1 values
+            ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a )
+            ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a )
+            ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a )
+            ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a )
+            ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a )
+            ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a )
+            ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" )
+            ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" )
+            ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" )
+            ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL )
+            '''
+        )
+
     def check_result_auto(self ,origin_query , abs_query):
         abs_result = tdSql.getResult(abs_query)
         origin_result = tdSql.getResult(origin_query)
@@ -95,6 +149,7 @@ class TDTestCase:
             tdLog.info("abs value check pass , it work as expected ,sql is \"%s\"   "%abs_query )
         
     def test_errors(self):
+        tdSql.execute("use testdb")
         error_sql_lists = [
             "select abs from t1",
             # "select abs(-+--+c1) from t1",
@@ -129,11 +184,16 @@ class TDTestCase:
             tdSql.error(error_sql)
     
     def support_types(self):
+        tdSql.execute("use testdb")
         type_error_sql_lists = [
             "select abs(ts) from t1" , 
+            "select abs(t0) from t1" , 
             "select abs(c7) from t1",
             "select abs(c8) from t1",
             "select abs(c9) from t1",
+            "select abs(t7) from t1",
+            "select abs(t8) from t1",
+            "select abs(t9) from t1",
             "select abs(ts) from ct1" , 
             "select abs(c7) from ct1",
             "select abs(c8) from ct1",
@@ -171,6 +231,13 @@ class TDTestCase:
             "select abs(c5) from t1",
             "select abs(c6) from t1",
 
+            "select abs(t1) from ct1",
+            "select abs(t2) from ct1",
+            "select abs(t3) from ct1",
+            "select abs(t4) from ct1",
+            "select abs(t5) from ct1",
+            "select abs(t6) from ct1",
+
             "select abs(c1) from ct1",
             "select abs(c2) from ct1",
             "select abs(c3) from ct1",
@@ -447,6 +514,38 @@ class TDTestCase:
 
         self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select abs(c1+1) ,abs(c2) , abs(c3*1) , abs(c4/2), abs(c5)/2, abs(c6) from sub1_bound ")
         
+    def test_tag_compute_for_scalar_function(self):
+        
+        tdSql.execute("use testdb")
+
+        self.check_result_auto( "select c1, t2, t3 , t4, t5 from ct4 ", "select (c1), abs(t2) ,abs(t3), abs(t4), abs(t5) from ct4")
+        self.check_result_auto( "select c1+2, t2+2, t3 , t4, t5 from ct4 ", "select (c1)+2, abs(t2)+2 ,abs(t3), abs(t4), abs(t5) from ct4")
+        self.check_result_auto( "select c1+2, t2+2, t3 , t4, t5 from stb1 order by t1 ", "select (c1)+2, abs(t2)+2 ,abs(t3), abs(t4), abs(t5) from stb1 order by t1")
+
+        # bug need fix 
+
+        # tdSql.query(" select sum(c1) from stb1 where t1+10 >1; ")  #　taosd crash 
+        tdSql.query("select c1 ,t1 from stb1 where t1 =0 ")
+        tdSql.checkRows(13)
+        # tdSql.query("select t1 from stb1 where t1 >0 ")
+        # tdSql.checkRows(3)
+        # tdSql.query("select sum(t1) from (select c1 ,t1 from stb1)")
+        # tdSql.checkData(0,0,61)
+        # tdSql.query("select distinct(c1) ,t1 from stb1")
+        # tdSql.checkRows(11)
+        # tdSql.query("select max(t2) , t1 ,c1, t2 from stb1")
+        # tdSql.checkData(0,3,33333)
+
+        # tag filter with abs function
+        # tdSql.query("select t1 from stb1 where abs(t1)=1")
+        # tdSql.checkRows(1)
+        tdSql.query("select t1 from stb1 where abs(c1+t1)=1")
+        tdSql.checkRows(1)
+        # tdSql.query("select t1 from stb1 where abs(t1+c1)=1")
+        # tdSql.checkRows(1)
+        tdSql.query("select abs(c1+t1)*t1 from stb1 where abs(c1)/floor(abs(ceil(t1))) ==1")
+
+
 
     def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
         tdSql.prepare()
@@ -454,6 +553,7 @@ class TDTestCase:
         tdLog.printNoPrefix("==========step1:create table ==============")
         
         self.prepare_datas()
+        self.prepare_tag_datas()
 
         tdLog.printNoPrefix("==========step2:test errors ==============")    
 
@@ -475,6 +575,10 @@ class TDTestCase:
 
         self.abs_func_filter()
 
+        tdLog.printNoPrefix("==========step6: tag coumpute query ============") 
+        
+        self.test_tag_compute_for_scalar_function()
+
     def stop(self):
         tdSql.close()
         tdLog.success(f"{__file__} successfully executed")

tests/system-test/fulltest.bat

 
-@REM python3 .\test.py -f 0-others\taosShell.py
-@REM python3 .\test.py -f 0-others\taosShellError.py
+python3 .\test.py -f 0-others\taosShell.py
+python3 .\test.py -f 0-others\taosShellError.py
 python3 .\test.py -f 0-others\taosShellNetChk.py
 python3 .\test.py -f 0-others\telemetry.py
 python3 .\test.py -f 0-others\taosdMonitor.py
 python3 .\test.py -f 0-others\udfTest.py
 python3 .\test.py -f 0-others\udf_create.py
 @REM python3 .\test.py -f 0-others\udf_restart_taosd.py
-@REM python3 .\test.py -f 0-others\cachelast.py
+python3 .\test.py -f 0-others\cachelast.py
 
-@REM python3 .\test.py -f 0-others\user_control.py
-@REM python3 .\test.py -f 0-others\fsync.py
+python3 .\test.py -f 0-others\user_control.py
+python3 .\test.py -f 0-others\fsync.py
 
-@REM python3 .\test.py -f 1-insert\influxdb_line_taosc_insert.py
-@REM python3 .\test.py -f 1-insert\opentsdb_telnet_line_taosc_insert.py
-@REM python3 .\test.py -f 1-insert\opentsdb_json_taosc_insert.py
+python3 .\test.py -f 1-insert\influxdb_line_taosc_insert.py
+python3 .\test.py -f 1-insert\opentsdb_telnet_line_taosc_insert.py
+python3 .\test.py -f 1-insert\opentsdb_json_taosc_insert.py
 @REM #python3 .\test.py -f 1-insert\test_stmt_muti_insert_query.py
-@REM python3 .\test.py -f 1-insert\alter_stable.py
-@REM python3 .\test.py -f 1-insert\alter_table.py
-@REM python3 .\test.py -f 2-query\between.py
-@REM python3 .\test.py -f 2-query\distinct.py
-@REM python3 .\test.py -f 2-query\varchar.py
+python3 .\test.py -f 1-insert\alter_stable.py
+python3 .\test.py -f 1-insert\alter_table.py
+python3 .\test.py -f 2-query\between.py
+python3 .\test.py -f 2-query\distinct.py
+python3 .\test.py -f 2-query\varchar.py
 @REM python3 .\test.py -f 2-query\ltrim.py
-@REM python3 .\test.py -f 2-query\rtrim.py
-@REM python3 .\test.py -f 2-query\length.py
-@REM python3 .\test.py -f 2-query\char_length.py
-@REM python3 .\test.py -f 2-query\upper.py
-@REM python3 .\test.py -f 2-query\lower.py
-@REM python3 .\test.py -f 2-query\join.py
-@REM python3 .\test.py -f 2-query\join2.py
-@REM python3 .\test.py -f 2-query\cast.py
-@REM python3 .\test.py -f 2-query\union.py
-@REM python3 .\test.py -f 2-query\union1.py
+python3 .\test.py -f 2-query\rtrim.py
+python3 .\test.py -f 2-query\length.py
+python3 .\test.py -f 2-query\char_length.py
+python3 .\test.py -f 2-query\upper.py
+python3 .\test.py -f 2-query\lower.py
+python3 .\test.py -f 2-query\join.py
+python3 .\test.py -f 2-query\join2.py
+python3 .\test.py -f 2-query\cast.py
+python3 .\test.py -f 2-query\union.py
+python3 .\test.py -f 2-query\union1.py
 @REM python3 .\test.py -f 2-query\concat.py
-@REM python3 .\test.py -f 2-query\concat2.py
-@REM python3 .\test.py -f 2-query\concat_ws.py
-@REM python3 .\test.py -f 2-query\concat_ws2.py
+python3 .\test.py -f 2-query\concat2.py
+python3 .\test.py -f 2-query\concat_ws.py
+python3 .\test.py -f 2-query\concat_ws2.py
 @REM python3 .\test.py -f 2-query\check_tsdb.py
 @REM python3 .\test.py -f 2-query\spread.py
 @REM python3 .\test.py -f 2-query\hyperloglog.py
 
+python3 .\test.py -f 2-query\timezone.py
+python3 .\test.py -f 2-query\Now.py
+python3 .\test.py -f 2-query\Today.py
+python3 .\test.py -f 2-query\max.py
+python3 .\test.py -f 2-query\min.py
+python3 .\test.py -f 2-query\count.py
+python3 .\test.py -f 2-query\last.py
+python3 .\test.py -f 2-query\first.py
+python3 .\test.py -f 2-query\To_iso8601.py
+python3 .\test.py -f 2-query\To_unixtimestamp.py
+python3 .\test.py -f 2-query\timetruncate.py
+python3 .\test.py -f 2-query\diff.py
+python3 .\test.py -f 2-query\Timediff.py
 
-@REM python3 .\test.py -f 2-query\timezone.py
-@REM python3 .\test.py -f 2-query\Now.py
-@REM python3 .\test.py -f 2-query\Today.py
-@REM python3 .\test.py -f 2-query\max.py
-@REM python3 .\test.py -f 2-query\min.py
-@REM python3 .\test.py -f 2-query\count.py
-@REM python3 .\test.py -f 2-query\last.py
-@REM python3 .\test.py -f 2-query\first.py
-@REM python3 .\test.py -f 2-query\To_iso8601.py
-@REM python3 .\test.py -f 2-query\To_unixtimestamp.py
-@REM python3 .\test.py -f 2-query\timetruncate.py
-@REM python3 .\test.py -f 2-query\diff.py
-@REM python3 .\test.py -f 2-query\Timediff.py
-
-@REM python3 .\test.py -f 2-query\top.py
-@REM python3 .\test.py -f 2-query\bottom.py
-@REM python3 .\test.py -f 2-query\percentile.py
-@REM python3 .\test.py -f 2-query\apercentile.py
-@REM python3 .\test.py -f 2-query\abs.py
-@REM python3 .\test.py -f 2-query\ceil.py
-@REM python3 .\test.py -f 2-query\floor.py
-@REM python3 .\test.py -f 2-query\round.py
-@REM python3 .\test.py -f 2-query\log.py
-@REM python3 .\test.py -f 2-query\pow.py
-@REM python3 .\test.py -f 2-query\sqrt.py
-@REM python3 .\test.py -f 2-query\sin.py
-@REM python3 .\test.py -f 2-query\cos.py
-@REM python3 .\test.py -f 2-query\tan.py
-@REM python3 .\test.py -f 2-query\arcsin.py
-@REM python3 .\test.py -f 2-query\arccos.py
-@REM python3 .\test.py -f 2-query\arctan.py
+python3 .\test.py -f 2-query\top.py
+python3 .\test.py -f 2-query\bottom.py
+python3 .\test.py -f 2-query\percentile.py
+python3 .\test.py -f 2-query\apercentile.py
+python3 .\test.py -f 2-query\abs.py
+python3 .\test.py -f 2-query\ceil.py
+python3 .\test.py -f 2-query\floor.py
+python3 .\test.py -f 2-query\round.py
+python3 .\test.py -f 2-query\log.py
+python3 .\test.py -f 2-query\pow.py
+python3 .\test.py -f 2-query\sqrt.py
+python3 .\test.py -f 2-query\sin.py
+python3 .\test.py -f 2-query\cos.py
+python3 .\test.py -f 2-query\tan.py
+python3 .\test.py -f 2-query\arcsin.py
+python3 .\test.py -f 2-query\arccos.py
+python3 .\test.py -f 2-query\arctan.py
 @REM python3 .\test.py -f 2-query\query_cols_tags_and_or.py
 @REM # python3 .\test.py -f 2-query\nestedQuery.py
 @REM # TD-15983 subquery output duplicate name column. 
 @REM # Please Xiangyang Guo modify the following script
 @REM # python3 .\test.py -f 2-query\nestedQuery_str.py
 
-@REM python3 .\test.py -f 2-query\avg.py
-@REM python3 .\test.py -f 2-query\elapsed.py
+python3 .\test.py -f 2-query\avg.py
+python3 .\test.py -f 2-query\elapsed.py
 @REM python3 .\test.py -f 2-query\csum.py
-@REM python3 .\test.py -f 2-query\mavg.py
-@REM python3 .\test.py -f 2-query\diff.py
-@REM python3 .\test.py -f 2-query\sample.py
+python3 .\test.py -f 2-query\mavg.py
+python3 .\test.py -f 2-query\diff.py
+python3 .\test.py -f 2-query\sample.py
 @REM python3 .\test.py -f 2-query\function_diff.py
-@REM python3 .\test.py -f 2-query\unique.py
-@REM python3 .\test.py -f 2-query\stateduration.py
-@REM python3 .\test.py -f 2-query\function_stateduration.py
-@REM python3 .\test.py -f 2-query\statecount.py
+python3 .\test.py -f 2-query\unique.py
+python3 .\test.py -f 2-query\stateduration.py
+python3 .\test.py -f 2-query\function_stateduration.py
+python3 .\test.py -f 2-query\statecount.py
 
 @REM python3 .\test.py -f 7-tmq\basic5.py
 @REM python3 .\test.py -f 7-tmq\subscribeDb.py
@@ -100,4 +99,4 @@ python3 .\test.py -f 0-others\udf_create.py
 @REM python3 .\test.py -f 7-tmq\subscribeStb2.py
 @REM python3 .\test.py -f 7-tmq\subscribeStb3.py
 @REM python3 .\test.py -f 7-tmq\subscribeStb4.py
-@REM python3 .\test.py -f 7-tmq\db.py
+@REM python3 .\test.py -f 7-tmq\db.py
\ No newline at end of file

tests/system-test/simpletest.bat

+
+@REM python3 .\test.py -f 0-others\taosShell.py
+@REM python3 .\test.py -f 0-others\taosShellError.py
+python3 .\test.py -f 0-others\taosShellNetChk.py
+python3 .\test.py -f 0-others\telemetry.py
+python3 .\test.py -f 0-others\taosdMonitor.py
+python3 .\test.py -f 0-others\udfTest.py
+python3 .\test.py -f 0-others\udf_create.py
+@REM python3 .\test.py -f 0-others\udf_restart_taosd.py
+@REM python3 .\test.py -f 0-others\cachelast.py
+
+@REM python3 .\test.py -f 0-others\user_control.py
+@REM python3 .\test.py -f 0-others\fsync.py
+
+@REM python3 .\test.py -f 1-insert\influxdb_line_taosc_insert.py
+@REM python3 .\test.py -f 1-insert\opentsdb_telnet_line_taosc_insert.py
+@REM python3 .\test.py -f 1-insert\opentsdb_json_taosc_insert.py
+@REM #python3 .\test.py -f 1-insert\test_stmt_muti_insert_query.py
+@REM python3 .\test.py -f 1-insert\alter_stable.py
+@REM python3 .\test.py -f 1-insert\alter_table.py
+@REM python3 .\test.py -f 2-query\between.py
+@REM python3 .\test.py -f 2-query\distinct.py
+@REM python3 .\test.py -f 2-query\varchar.py
+@REM python3 .\test.py -f 2-query\ltrim.py
+@REM python3 .\test.py -f 2-query\rtrim.py
+@REM python3 .\test.py -f 2-query\length.py
+@REM python3 .\test.py -f 2-query\char_length.py
+@REM python3 .\test.py -f 2-query\upper.py
+@REM python3 .\test.py -f 2-query\lower.py
+@REM python3 .\test.py -f 2-query\join.py
+@REM python3 .\test.py -f 2-query\join2.py
+@REM python3 .\test.py -f 2-query\cast.py
+@REM python3 .\test.py -f 2-query\union.py
+@REM python3 .\test.py -f 2-query\union1.py
+@REM python3 .\test.py -f 2-query\concat.py
+@REM python3 .\test.py -f 2-query\concat2.py
+@REM python3 .\test.py -f 2-query\concat_ws.py
+@REM python3 .\test.py -f 2-query\concat_ws2.py
+@REM python3 .\test.py -f 2-query\check_tsdb.py
+@REM python3 .\test.py -f 2-query\spread.py
+@REM python3 .\test.py -f 2-query\hyperloglog.py
+
+
+@REM python3 .\test.py -f 2-query\timezone.py
+@REM python3 .\test.py -f 2-query\Now.py
+@REM python3 .\test.py -f 2-query\Today.py
+@REM python3 .\test.py -f 2-query\max.py
+@REM python3 .\test.py -f 2-query\min.py
+@REM python3 .\test.py -f 2-query\count.py
+@REM python3 .\test.py -f 2-query\last.py
+@REM python3 .\test.py -f 2-query\first.py
+@REM python3 .\test.py -f 2-query\To_iso8601.py
+@REM python3 .\test.py -f 2-query\To_unixtimestamp.py
+@REM python3 .\test.py -f 2-query\timetruncate.py
+@REM python3 .\test.py -f 2-query\diff.py
+@REM python3 .\test.py -f 2-query\Timediff.py
+
+@REM python3 .\test.py -f 2-query\top.py
+@REM python3 .\test.py -f 2-query\bottom.py
+@REM python3 .\test.py -f 2-query\percentile.py
+@REM python3 .\test.py -f 2-query\apercentile.py
+@REM python3 .\test.py -f 2-query\abs.py
+@REM python3 .\test.py -f 2-query\ceil.py
+@REM python3 .\test.py -f 2-query\floor.py
+@REM python3 .\test.py -f 2-query\round.py
+@REM python3 .\test.py -f 2-query\log.py
+@REM python3 .\test.py -f 2-query\pow.py
+@REM python3 .\test.py -f 2-query\sqrt.py
+@REM python3 .\test.py -f 2-query\sin.py
+@REM python3 .\test.py -f 2-query\cos.py
+@REM python3 .\test.py -f 2-query\tan.py
+@REM python3 .\test.py -f 2-query\arcsin.py
+@REM python3 .\test.py -f 2-query\arccos.py
+@REM python3 .\test.py -f 2-query\arctan.py
+@REM python3 .\test.py -f 2-query\query_cols_tags_and_or.py
+@REM # python3 .\test.py -f 2-query\nestedQuery.py
+@REM # TD-15983 subquery output duplicate name column. 
+@REM # Please Xiangyang Guo modify the following script
+@REM # python3 .\test.py -f 2-query\nestedQuery_str.py
+
+@REM python3 .\test.py -f 2-query\avg.py
+@REM python3 .\test.py -f 2-query\elapsed.py
+@REM python3 .\test.py -f 2-query\csum.py
+@REM python3 .\test.py -f 2-query\mavg.py
+@REM python3 .\test.py -f 2-query\diff.py
+@REM python3 .\test.py -f 2-query\sample.py
+@REM python3 .\test.py -f 2-query\function_diff.py
+@REM python3 .\test.py -f 2-query\unique.py
+@REM python3 .\test.py -f 2-query\stateduration.py
+@REM python3 .\test.py -f 2-query\function_stateduration.py
+@REM python3 .\test.py -f 2-query\statecount.py
+
+@REM python3 .\test.py -f 7-tmq\basic5.py
+@REM python3 .\test.py -f 7-tmq\subscribeDb.py
+@REM python3 .\test.py -f 7-tmq\subscribeDb0.py
+@REM python3 .\test.py -f 7-tmq\subscribeDb1.py
+@REM python3 .\test.py -f 7-tmq\subscribeStb.py
+@REM python3 .\test.py -f 7-tmq\subscribeStb0.py
+@REM python3 .\test.py -f 7-tmq\subscribeStb1.py
+@REM python3 .\test.py -f 7-tmq\subscribeStb2.py
+@REM python3 .\test.py -f 7-tmq\subscribeStb3.py
+@REM python3 .\test.py -f 7-tmq\subscribeStb4.py
+@REM python3 .\test.py -f 7-tmq\db.py

tests/system-test/test-all.bat

@@ -2,26 +2,42 @@
 SETLOCAL EnableDelayedExpansion
 for /F "tokens=1,2 delims=#" %%a in ('"prompt #$H#$E# & echo on & for %%b in (1) do     rem"') do (  set "DEL=%%a")
 set /a a=0
+if %1 == full (
+    echo Windows Taosd Full Test
+    set /a exitNum=0
+    for /F "usebackq tokens=*" %%i in (fulltest.bat) do (
+        for /f "tokens=1* delims= " %%a in ("%%i") do if not "%%a" == "@REM" (
+            set /a a+=1
+            echo !a! Processing %%i
+            call :GetTimeSeconds !time!
+            set time1=!_timeTemp!
+            echo Start at !time!
+            call %%i ARG1 > result_!a!.txt 2>error_!a!.txt
+            if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 ) else ( call :colorEcho 0a "Success" &echo. ) 
+        )
+    )
+    exit !exitNum!
+)
 echo Windows Taosd Test
-for /F "usebackq tokens=*" %%i in (fulltest.bat) do (
+for /F "usebackq tokens=*" %%i in (simpletest.bat) do (
     for /f "tokens=1* delims= " %%a in ("%%i") do if not "%%a" == "@REM" (
-        echo Processing %%i
-        call :GetTimeSeconds %time%
-        set time1=!_timeTemp!
-        echo Start at %time%
         set /a a+=1
+        echo !a! Processing %%i
+        call :GetTimeSeconds !time!
+        set time1=!_timeTemp!
+        echo Start at !time!
         call %%i ARG1 > result_!a!.txt 2>error_!a!.txt
         if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && echo result: && cat result_!a!.txt && echo error: && cat error_!a!.txt && exit 8 ) else ( call :colorEcho 0a "Success" &echo. ) 
     )
 )
 @REM echo Linux Taosd Test
-@REM for /F "usebackq tokens=*" %%i in (fulltest.bat) do (
+@REM for /F "usebackq tokens=*" %%i in (simpletest.bat) do (
 @REM     for /f "tokens=1* delims= " %%a in ("%%i") do if not "%%a" == "@REM" (
-@REM         echo Processing %%i
-@REM         call :GetTimeSeconds %time%
-@REM         set time1=!_timeTemp!
-@REM         echo Start at %time%
 @REM         set /a a+=1
+@REM         echo !a! Processing %%i
+@REM         call :GetTimeSeconds !time!
+@REM         set time1=!_timeTemp!
+@REM         echo Start at !time!
 @REM         call %%i ARG1 -m %1 > result_!a!.txt 2>error_!a!.txt
 @REM         if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && echo result: && cat result_!a!.txt && echo error: && cat error_!a!.txt && exit 8 ) else ( call :colorEcho 0a "Success" &echo. ) 
 @REM     )
@@ -57,5 +73,5 @@ for %%a in (%tt%) do (
     )
    set /a index=index+1
 )
-set /a _timeTemp=(%hh%*60+%mm%)*60+%ss%
+set /a _timeTemp=(%hh%*60+%mm%)*60+%ss%  || echo hh:%hh% mm:%mm% ss:%ss%
 goto :eof
\ No newline at end of file

taos-tools @ 717f5aaa

-Subproject commit 932da0f4cac013c2eded824d1d4d01cfa6168fa3
+Subproject commit 717f5aaa5f0a1b4d92bb2ae68858fec554fb5eda